I have been trying to provide the functionality to stop a thread following this thread . I have the following class:
class WorkerThread(threading.Thread):
"""Thread class with a stop() method. The thread itself has to check
regularly for the stopped() condition."""
def __init__(self, id):
super(WorkerThread, self).__init__()
self._id = id
self._stop = threading.Event()
def stop(self):
self._stop.set()
def stopped(self):
return self._stop.isSet()
def run(self):
while not self.stopped():
print ("Thread with id {0} Running".format(self._id))
time.sleep(1)
When I create 10 threads and try to stop them, none of them stop and keep printing.
threads = [WorkerThread(x) for x in range(10)]
[thread.start() for thread in threads]
time.sleep(2) #wait
[thread.stop() for thread in threads]
[thread.join() for thread in threads]
print "All done" #never printed
Can someone explain why the stop event is ignored? I also tried using the _stop attribute as a flag and setting it to false in __init__ and True in stop() with no luck.
Related
I have a script which uses threads, but it is unable to catch Ctrl + C.
Here it is the sample code to reproduce this error:
import threading
import time
import signal
class DummyThread(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self._running = True
signal.signal(signal.SIGINT, self.stop)
signal.signal(signal.SIGTERM, self.stop)
def stop(self, signum=None, frame=None):
self._running = False
def run(self):
while self._running:
time.sleep(1)
print("Running")
if __name__ == "__main__":
try:
t = DummyThread()
t.start()
while True:
print("Main thread running")
time.sleep(0.5)
except KeyboardInterrupt:
print("This never gets printed")
t.stop()
finally:
print("Exit")
When I run python3 script.py it starts running, but it does not catch ctrl+c. I have googled it but I have not found a solution. I have to kill the script with SIGTERM, but I want DummyThread to stop gracefully.
class DummyThread(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self._running = True
signal.signal(signal.SIGINT, self.stop)
signal.signal(signal.SIGTERM, self.stop)
The program actually does not work as expected because of those last two lines and would work without them.
The reason is that, if you press Ctrl-C, the SIGINT signal is handled by the signal handler that is set up by signal.signal and self.stop is called. So the thread should actually stop.
But in the main thread, the while True loop is still running. Since the signal has already been handled, there will be no KeyboardInterrupt exception raised by the Python runtime. Therefore you never get to the except part.
if __name__ == "__main__":
try:
t = DummyThread()
t.start()
while True: # you are stuck in this loop
print("Main thread running")
time.sleep(0.5)
except KeyboardInterrupt: # this never happens
print("This never gets printed")
t.stop()
Only one signal handler should be set up to call the stop method. So there are two options to solve the problem:
Handle the signal implicitly by catching the KeyboardInterrupt exception. This is achieved by simply removing the two signal.signal(...) lines.
Set up an explicit signal handler (as you did by using signal.signal in DummyThread.__init__), but remove the while True: loop from the main thread and do not try to handle KeyboardInterrupt. Instead, just wait for the DummyThread to finish on its own by using its join method:
if __name__ == "__main__":
t = DummyThread()
t.start()
t.join()
print("Exit")
The main point is that you can't work with signals in any other Thread except the Main Thread. The Main Thread is the only one which can receive signals and handle them. I can offer the following solution, it is based on Event sync primitive.
According to Python documantation:
Signals and threads
Python signal handlers are always executed in the main Python thread, even if the signal was received in another thread. This means that signals can’t be used as a means of inter-thread communication. You can use the synchronization primitives from the threading module instead.
Besides, only the main thread is allowed to set a new signal handler.
from threading import Thread, Event
import time
class DummyThread(Thread):
def __init__(self, event: Event):
Thread.__init__(self)
self.stop_event = event
def run(self):
# we are monitoring the event in the Main Thread
while not self.stop_event.is_set():
time.sleep(1)
print("Running")
# only Main Thread can make the point reachable
print("I am done !")
if __name__ == "__main__":
try:
e = Event()
t = DummyThread(e)
t.start()
while True:
print("Main thread running")
time.sleep(0.5)
except KeyboardInterrupt:
e.set()
finally:
print("Exit")
Another possible choice is to use daemon Thread for such tasks like in your code example (when you just printing smth in the screen every second, rather than e.g. close database connection or some similar task). If main thread is stoped the daemon Thread will stop too.
As shown in your code, you used KeyboardInterrupt to call stop() function. See how Listener does the same task and stops the execution which was unable to catch from Ctrl + C. You dont have to kill the script with SIGTERM anymore
import threading
import time
import signal
import os
from pynput.keyboard import Key, Listener
class DummyThread(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self._running = True
signal.signal(signal.SIGINT, self.stop)
signal.signal(signal.SIGTERM, self.stop)
def stop(self, signum=None, frame=None):
self._running = False
print ("Bye Bye . .")
os._exit(1)
def run(self):
while self._running:
time.sleep(1)
print("Running")
if __name__ == "__main__":
t = DummyThread()
def func2():
try:
t.start()
while True:
print("Main thread running")
time.sleep(0.5)
except KeyboardInterrupt:
print("No need for this")
t.stop()
finally:
print("Exit")
def func1():
with Listener(on_press = t.stop) as listener :
listener.join()
threading.Thread(target=func1).start()
threading.Thread(target=func2).start()
I am writing multiprocess program. There are four class: Main, Worker, Request and Ack. The Main class is the entry point of program. It will create the sub-process called Worker to do some jobs. The main process put the Request into JoinableQueue, and than Worker get request from queue. When Worker finished the request, it will put the ACK into queue. The part of code shown as below:
Main:
class Main():
def __init__(self):
self.cmd_queue = JoinableQueue()
self.worker = Worker(self.cmd_queue)
def call_worker(self, cmd_code):
if self.cmd_queue.empty() is True:
request = Request(cmd_code)
self.cmd_queue.put(request)
self.cmd_queue.join()
ack = self.cmd_queue.get()
self.cmd_queue.task_done()
if ack.value == 0:
return True
else:
return False
else:
# TODO: Error Handling.
pass
def run_worker(self):
self.worker.start()
Worker:
class Worker(Process):
def __init__(self, cmd_queue):
super(Worker, self).__init__()
self.cmd_queue = cmd_queue
...
def run(self):
while True:
ack = Ack(0)
try:
request = self.cmd_queue.get()
if request.cmd_code == ReqCmd.enable_handler:
self.enable_handler()
elif request.cmd_code == ReqCmd.disable_handler:
self.disable_handler()
else:
pass
except Exception:
ack.value = -1
finally:
self.cmd_queue.task_done()
self.cmd_queue.put(ack)
self.cmd_queue.join()
It often works normally. But Main process stuck at self.cmd_queue.join(), and the Worker stuck at self.cmd_queue.join() sometimes. It is so weird! Does anyone have any ideas? Thanks
There's nothing weird in the above issue: you shouldn't call queue's join within a typical single worker process activity because
Queue.join()
Blocks until all items in the queue have been gotten and
processed.
Such a calls where they are in your current implementation will make the processing pipeline wait.
Usually queue.join() is called in the main (supervisor) thread after initiating/starting all threads/workers.
https://docs.python.org/3/library/queue.html#queue.Queue.join
I have my own Thread called TimeBasedLogThread. I would like to fire a function my_function when the TimeBasedLogThread is being killed because the main process is exiting. I would like to do it from within this object. Is it possible to do so?
Here is my current approach:
class TimeBasedBufferingHandler(MemoryHandler):
# This is a logging-based handler that buffers logs to send
# them as emails
# the target of this handler is a SMTPHandler
def __init__(self, capacity=10, flushLevel=logging.ERROR, target=None,
flushOnClose=True, timeout=60):
MemoryHandler.__init__(self, capacity=capacity, flushLevel=flushLevel,
target=target, flushOnClose=flushOnClose)
self.timeout = timeout # in seconds (as time.time())
def flush(self):
# Send the emails that are younger than timeout, all together
# in the same email
class TimeBasedLogThread(Thread):
def __init__(self, handler, timeout=60):
Thread.__init__(self)
self.handler = handler
self.timeout = timeout
def run(self):
while True:
self.handler.flush()
time.sleep(self.timeout)
def my_function(self):
print("my_function is being called")
self.handler.flush()
def setup_thread():
smtp_handler = SMTPHandler()
new_thread = TimeBasedLogThread(smtp_handler, timeout=10)
new_thread.start()
In my main thread, I have:
setup_thread()
logging.error("DEBUG_0")
time.sleep(5)
logging.error("DEBUG_1")
time.sleep(5)
logging.error("DEBUG_2")
The time.sleep(5) releases the main thread 5 seconds before the timeout of my other thread. So, I receive the first 2 emails with "DEBUG_0" and "DEBUG_1", but not the last one "DEBUG_2" because the main process exits before the timeout has finished.
I would like to link the class TimeBasedLogThread and the function my_function that will flush (send the emails) before exiting. How can I do that? I looked at the source code of threading but I did not understand what method I could use.
Build your function as a Thread too. (Ex: AfterDeadThread)
You have two strategy here:
TimeBasedLogThread call AfterDeadThread before die
AfterDeadThread check if TimeBasedLogThread is alive or not, if not it will run some methods
Extend run() method (representing the thread’s activity) to fire the on_terminate handler passed to custom thread’s constructor as keyword argument.
On a slightly changed custom thread class (for demonstration):
from threading import Thread
import time, random
class TimeBasedLogThread(Thread):
def __init__(self, handler, timeout=2, on_terminate=None):
Thread.__init__(self)
self.handler = handler
self.timeout = timeout
self.terminate_handler = on_terminate
def run(self):
while True:
num = self.handler()
if num > 5:
break
time.sleep(self.timeout)
print(num)
if self.terminate_handler:
self.terminate_handler()
def my_term_function():
print("my_function is being called")
f = lambda: random.randint(3, 10)
tlog_thread = TimeBasedLogThread(f, on_terminate=my_term_function)
tlog_thread.start()
tlog_thread.join()
Sample output:
3
4
5
4
5
my_function is being called
I am trying to create an abort button in a GUI for stopping threads that are blocking on tcl.eval().
Class Application:
def __init__(self):
self.tcl = Tkinter.Tcl()
def eval(self, cmd):
return self.tcl.eval(cmd)
def stop(self):
# How can I interrupt a thread which is blocking in
# self.eval() from here?
pass
From my understanding there is no such out-of-the-box solution in the python stdlib.
The solution has to have the following characteristics:
Start the thread as daemon with a target function and optional arguments
Have polling: the thread should rerun the target every X seconds
Allow easy and graceful stopping: Not breaking target execution midway
Expose the ability to stop from outside the program it belongs to, in particular be able to stop the thread from testing code.
Allow thread restarting after stop (or pseudo-restarting with a new thread)
I ran across a couple of suggestions on SO, but would like to aggregate any collected knowledge here (i will contribute in that in a follow-up answer), so that any new or alternative or additional ideas be heard.
My proposal uses the threading library, as it is advertised as more high level than thread.
A middle ground is this solution, found from other SO answer:
def main():
t_stop= threading.Event()
t = threading.Thread(target=thread, args=(1, t_stop))
t.daemon = True
t.start()
time.sleep(duration)
#stop the thread
t_stop.set()
def thread(arg, stop_event):
while(not stop_event.is_set()):
# Code to execute here
stop_event.wait(time)
This, unfortunately, requires us to have the t_stop object handy when testing -in order to stop the thread- and that handle to the object is not designed to be exposed.
A solution would be to add t and t_stop handles in a top level or global dictionary somewhere, for the testing code to reach.
Another solution (copied and improved from somewhere) is use of the following:
def main():
t = DaemonStoppableThread(sleep_time, target=target_function,
name='polling_thread',
args=(arg1, arg2))
t.start()
# Stopping code from a test
def stop_polling_threads():
threads = threading.enumerate()
polling_threads = [thread for thread in threads
if 'polling_thread' in thread.getName()]
for poll_thread in polling_threads:
poll_thread.stop()
class DaemonStoppableThread(threading.Thread):
def __init__(self, sleep_time, target=None, **kwargs):
super(DaemonStoppableThread, self).__init__(target=target, **kwargs)
self.setDaemon(True)
self.stop_event = threading.Event()
self.sleep_time = sleep_time
self.target = target
def stop(self):
self.stop_event.set()
def stopped(self):
return self.stop_event.isSet()
def run(self):
while not self.stopped():
if self.target:
self.target()
else:
raise Exception('No target function given')
self.stop_event.wait(self.sleep_time)
As good as these solutions may be, none of them face the restarting of the polling target function.
I avoided using the expression "restarting thread", as I understand that python threads cannot be restarted, so a new thread will have to be used to allow for this "pseudo-restarting"
EDIT:
To improve on the above, a solution to start/stop the polling target multiple times:
class ThreadManager(object):
def __init__(self):
self.thread = None
def start_thread(self):
if not self.thread or not self.thread.is_alive():
self.thread = DaemonStoppableThread(sleep_time=5, target=some_func, args=(1, 2))
self.thread.start()
return 'thread running'
def stop_thread(self):
if self.thread and self.thread.is_alive():
self.thread.stop()
return 'thread stopped'
else:
return 'dead thread'
def check_thread(self):
if self.thread and self.thread.is_alive():
return 'thread alive'
else:
return 'dead_thread'