I'm running a python script that will restart a subprocess every 2 hours. code as follow.
import datetime
import os
import signal
import subprocess
import sys
import time
if __name__ == '__main__':
while True:
with subprocess.Popen([sys.executable, '/script.py']) as p:
start_time = datetime.datetime.now()
try:
print(f'starting the process at {p.pid}')
p.communicate(timeout=4800)
except subprocess.TimeoutExpired:
print('timeout terminate process')
os.kill(p.pid, signal.SIGINT)
p.terminate()
p.wait(60)
p.kill()
dif = datetime.datetime.now()-start_time
t = 4810-dif.total_seconds()
print(f'restarting after {t/60} mins')
time.sleep(t)
in my script.py, I'm having a thread executor that each thread runs a chrome webdriver instance. The issue I'm having is when the process timeout, the python process is terminated but all the webdriver instances are still lingering. Is there a way for the parent process to kill all the spawned child processes aka the chrome driver instances in my case? Running on my mac
Edit:
script.py
def run_with_config(user):
i = WebDriverInstance()
try:
return i.run_loop() # while true infinite loop running some ui actions
except Exception:
return False
finally:
i.quit()
def run(users):
with concurrent.futures.ThreadPoolExecutor(max_workers=len(user_configs)) as executor:
f_to_user = {}
for c in user_configs:
f = executor.submit(run_with_config, c)
f_to_user[f] = c
for f in concurrent.futures.as_completed(f_to_user):
res = f.result()
print(res)
Run the subprocess in a new process group, then kill the entire group with os.killpg().
if __name__ == '__main__':
while True:
with subprocess.Popen([sys.executable, '/script.py'], creationflags=subprocess.CREATE_NEW_PROCESS_GROUP) as p:
start_time = datetime.datetime.now()
try:
print(f'starting the process at {p.pid}')
p.communicate(timeout=4800)
except subprocess.TimeoutExpired:
print('timeout terminate process')
os.killpg(p.pid, signal.SIGINT)
p.terminate()
p.wait(60)
p.kill()
dif = datetime.datetime.now()-start_time
t = 4810-dif.total_seconds()
print(f'restarting after {t/60} mins')
time.sleep(t)
passing preexec_fn=os.setpgrp to subprocess popen works for me.
Related
I am trying to implement a method to force stop the child that have been started with ThreadPoolExecutor / ProcessPoolExecutor. I would like a cross platform implementation (Windows and Linux).
When the signal is triggered from main, the main process exits and I do NOT want that, only the child.
What is the correct way to force the child to quit? I do NOT want Events because in the following example I can have a while loop that never gets to event.is_set() again
eg:
while not event.is_set():
# Do stuff
while waiting_for_something:
# Here is blocked
Here is the code I am using but I miss something and I don't know what:
import os
import signal
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
import time
def handler(signum, frame):
print(signum, os.getpid())
os.kill(os.getpid(), signal.SIGINT)
class asd:
def __init__(self):
pass
def run(self):
signal.signal(signal.SIGBREAK, handler)
while True:
print('running thread', os.getpid())
time.sleep(1)
while True:
print('running 2 ', os.getpid())
time.sleep(1)
print("after while")
if __name__ == "__main__":
t1 = asd()
pool = ProcessPoolExecutor(max_workers=4)
# pool = ThreadPoolExecutor(max_workers=4)
pool.submit(t1.run)
print('running main', os.getpid())
time.sleep(3)
signal.raise_signal(signal.SIGBREAK)
while True:
print("after killing process")
time.sleep(1)
Thank you!
you are sending the signal to your main python process not to the children.
in order to send signals to your children you need their PID, which is not available using the concurrent module, instead you should use multiprocess.Pool, then you can get the PID of the children and send the signal to them using os.kill
just remember to eventually use pool.terminate() to guarantee resources cleanup.
import os
import signal
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
import time
import psutil
import multiprocessing
def handler(signum, frame):
print(signum, os.getpid())
os.kill(os.getpid(), signal.SIGINT)
class asd:
def __init__(self):
pass
def run(self):
signal.signal(signal.SIGBREAK, handler)
while True:
print('running thread', os.getpid())
time.sleep(1)
while True:
print('running 2 ', os.getpid())
time.sleep(1)
print("after while")
if __name__ == "__main__":
t1 = asd()
pool = multiprocessing.Pool(4)
children = multiprocessing.active_children()
res = pool.apply_async(t1.run)
print('running main', os.getpid())
time.sleep(3)
for child in children:
os.kill(child.pid,signal.SIGBREAK)
while True:
print("after killing process")
time.sleep(1)
with result
running main 16860
running thread 14212
running 2 14212
running 2 14212
after killing process
after killing process
after killing process
You can take a look at pebble which has been designed to solve this problem transparently for the User.
It provides concurrent.futures compatible APIs and allows to end a processing job either by cancelling the returned Future object or by setting a computing timeout.
import time
from pebble import ProcessPool
from concurrent.futures import TimeoutError
TIMEOUT = 5
def function(sleep):
while True:
time.sleep(sleep)
with ProcessPool() as pool:
future = pool.submit(function, TIMEOUT, 1)
assert isinstance(future.exception(), TimeoutError)
Note that you cannot stop executing threads in Python so only process pools can support such functionality.
I am trying to get started with multithreading in Python. I have multiple threads that acquire a lock, perform an operation, release the lock, and output the result to a csv file. The threads should be terminated in case the main thread finishes (e.g. through pressing Ctrl+C). How is this done?
I thought making the threads daemons should do the job, because "the entire Python program exits when only daemon threads are left" (official Python documentation). This is not sufficient. According to python: how to terminate a thread when main program ends, I then tried catching a KeyboardInterrupt and then terminating the threads manually. Also this is not working, it seems that the KeyboardInterrupt is not catched correctly.
It makes me think, there is something about multithreading in Python that I misunderstood... Find attached my code:
import random
import csv
import sys
from datetime import datetime
import threading
lock = threading.Lock()
def observer(obs):
print("In " + str(obs))
sys.stdout.flush()
with open("data/test_" + str(obs) + ".csv", 'a', newline='') as csvfile:
while True:
datenow = datetime.today()
lock.acquire()
print(str(obs) + "acquired")
sum = 0
for i in range(10000):
sum = sum + random.random()
print(str(obs) + "released")
sys.stdout.flush()
lock.release()
writer = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_ALL)
writer.writerow([datenow.isoformat(), sum/10000])
#print(str(obs) + ": " + datenow.isoformat() + " " + str(sum/1000))
sys.stdout.flush()
if __name__ == "__main__":
observe = [1, 2, 3, 4]
processes = []
for obs in observe:
process = threading.Thread(target=observer, args=(obs,), daemon=True)
processes.append(process)
print("Start processes")
for p in processes:
p.start()
print("Started")
try:
for p in processes:
p.join()
except KeyboardInterrupt:
for p in processes:
p.terminate()
print("Keyboard interrupt")
print("Finished")
Thanks!
This is wrong
try:
for p in processes:
p.join()
except KeyboardInterrupt:
for p in processes:
p.terminate()
print("Keyboard interrupt")
Your threads never exit the while True: loop, so you will be joining them forever. But that's not the point.
If Ctrl-C (aka KeyboardInterrupt) is not being caught my best bet:
You are running Python under Windows
The shell under which you run your script manipulates the processes and you actually never see the Ctrl-C because your program is being terminated abruptly.
You problem would then be:
Your program doesn't actually know it has terminated and that's why the threads hang.
If the assumptions made above are right, try this code (from Python - Windows - Exiting Child Process when "unrelated" parent dies/crashes)
import sys
def win_wait_for_parent(raise_exceptions=False):
if not sys.platform == 'win32':
return True
# When started under cygwin, the parent process will die leaving a child
# hanging around. The process has to be waited upon
import ctypes
from ctypes.wintypes import DWORD, BOOL, HANDLE
import os
import threading
INFINITE = -1
SYNCHRONIZE = 0x00100000
kernel32 = ctypes.WinDLL('kernel32', use_last_error=True)
kernel32.OpenProcess.argtypes = (DWORD, BOOL, DWORD)
kernel32.OpenProcess.restype = HANDLE
kernel32.WaitForSingleObject.argtypes = (HANDLE, DWORD)
kernel32.WaitForSingleObject.restype = DWORD
phandle = kernel32.OpenProcess(SYNCHRONIZE, 0, os.getppid())
def check_parent():
# Get a token with right access to parent and wait for it to be
# signaled (die). Exit ourselves then
kernel32.WaitForSingleObject(phandle, INFINITE)
os._exit(0)
if not phandle:
if raise_exceptions:
raise ctypes.WinError(ctypes.get_last_error())
return False
In Multiprocessing try to close console after terminating the process.
here is code
import multiprocessing
import time
# Your foo function
def foo(n):
for i in range(10000 * n):
print "Tick"
time.sleep(1)
if __name__ == '__main__':
p = multiprocessing.Process(target=foo, name="Foo", args=(10,))
p.start()
time.sleep(10)
# Terminate foo
p.terminate()
raise SystemExit
This code:
import multiprocessing as mp
from threading import Thread
import subprocess
import time
class WorkerProcess(mp.Process):
def run(self):
# Simulate long running task
self.subprocess = subprocess.Popen(['python', '-c', 'import time; time.sleep(1000)'])
self.code = self.subprocess.wait()
class ControlThread(Thread):
def run():
jobs = []
for _ in range(2):
job = WorkerProcess()
jobs.append(job)
job.start()
# wait for a while and then kill jobs
time.sleep(2)
for job in jobs:
job.terminate()
if __name__ == "__main__":
controller = ControlThread()
controller.start()
When I terminate the spawned WorkerProcess instances. They die just fine, however the subprocesses python -c 'import time; time.sleep(1000) runs until completition. This is well documented in the official docs, but how do I kill the child processes of a killed process?
A possbile soultion might be:
Wrap WorkerProcess.run() method inside try/except block catching SIGTERM, and terminating the subprocess.call call. But I am not sure how to catch the SIGTERM in the WorkerProcess
I also tried setting signal.signal(signal.SIGINT, handler) in the WorkerProcess, but I am getting ValuError, because it is allowed to be set only in the main thread.
What do I do now?
EDIT: As #svalorzen pointed out in comments this doesn't really work since the reference to self.subprocess is lost.
Finally came to a clean, acceptable solution. Since mp.Process.terminate is a method, we can override it.
class WorkerProcess(mp.Process):
def run(self):
# Simulate long running task
self.subprocess = subprocess.Popen(['python', '-c', 'import time; time.sleep(1000)'])
self.code = self.subprocess.wait()
# HERE
def terminate(self):
self.subprocess.terminate()
super(WorkerProcess, self).terminate()
You can use queues to message to your subprocesses and ask them nicely to terminate their children before exiting themselves. You can't use signals in anywhere else but your main thread, so signals are not suitable for this.
Curiously, when I modify the code like this, even if I interrupt it with control+C, subprocesses will die as well. This may be OS related thing, though.
import multiprocessing as mp
from threading import Thread
import subprocess
import time
from Queue import Empty
class WorkerProcess(mp.Process):
def __init__(self,que):
super(WorkerProcess,self).__init__()
self.queue = que
def run(self):
# Simulate long running task
self.subprocess = subprocess.Popen(['python', '-c', 'import time; time.sleep(1000)'])
while True:
a = self.subprocess.poll()
if a is None:
time.sleep(1)
try:
if self.queue.get(0) == "exit":
print "kill"
self.subprocess.kill()
self.subprocess.wait()
break
else:
pass
except Empty:
pass
print "run"
else:
print "exiting"
class ControlThread(Thread):
def run(self):
jobs = []
queues = []
for _ in range(2):
q = mp.Queue()
job = WorkerProcess(q)
queues.append(q)
jobs.append(job)
job.start()
# wait for a while and then kill jobs
time.sleep(5)
for q in queues:
q.put("exit")
time.sleep(30)
if __name__ == "__main__":
controller = ControlThread()
controller.start()
Hope this helps.
Hannu
I am looking for a python equivalent of following:
until python program.py; do
echo "Crashed...Restarting..." >&2
sleep 1
done
Also, I need to kill program.py when the parent program is killed. Any suggestions?
Modules subprocess and psutil should provide most (if not all) you need.
import sys, subprocess
while True :
retCode= subprocess.call(["python","program.py"])
if retCode == 0 : break
print('Crashed...Restarting...', file=sys.stderr )
#!/usr/bin/python
import os
import signal
import subprocess
import sys
import time
Proc = None
def signal_handler(sig, frame):
''' Kill the program '''
os.kill(Proc.pid, sig)
sys.exit(0)
def main():
global Proc
''' Handle signal on parent program '''
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGINT, signal_handler)
Proc = subprocess.Popen(['python', 'program.py'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
while True:
try:
wait = Proc.wait()
if wait:
if wait != (-1 * signal.SIGKILL):
print "Restarting ....(return code %d)" % wait
time.sleep(1)
Proc = subprocess.Popen(['python', 'program.py'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
else:
''' kill yourself '''
sys.exit(0)
except KeyboardInterrupt:
# If python >= 2.6
# Proc.kill()
sys.exit(0)
if __name__ == '__main__':
main()
If you can change program.py I would modify it so that you can call it directly, rather than in a subprocess. If your program follows the convention of containing only definitions and a final executable section to be executed only when it is directly called, i.e. it looks like
def do_something():
pass
def do_something_else():
pass
if __name__ == '__main__':
do_something()
do_something_else()
It is sufficient to wrap the last block in a function, such as
def main():
do_something()
do_something_else()
if __name__ == '__main__':
main()
At this point you can just import your program.py module and call program.main(), making your code much simpler.