Develop Reference

time.sleep(300) not allowing script to stop thread | python |

I am writing a code to insert data to mongodb every 5 minutes ON and OFF
The problem here is on keyword interrupt my thread should be stop and exit the code execution
Once first record is inserted to DB my time.sleep(300) will make the script sleep
and on my terminal the following line appears -> Press enter to kill the running thread :
Incase If I change my mind don't want to run
simply when I press enter from keyboard the threading which is running and under sleep should be stop and exit
My goal is to stop the thread on the basis of input from user
My code :
import datetime
import threading
import pymongo
import time
from pymongo import MongoClient
dbUrl = pymongo.MongoClient("mongodb://localhost:1245/")
dbName = dbUrl["student"]
dbCollectionName = dbName["student_course"]
def doremon():
return "Hi Friends"
def insert_to_mongodb():
global kill_the_running_thread
while (not kill_the_running_thread):
note_start_time = datetime.datetime.now()
msg = doremon()
note_end_time = datetime.datetime.now()
dt = {"message": msg, "start_time": note_start_time, "end_time": note_end_time}
rec_id1 = dbCollectionName.insert_one(dt)
time.sleep(300)
def main():
global kill_the_running_thread
kill_the_running_thread = False
my_thread = threading.Thread(target=insert_to_mongodb)
my_thread.start()
input("Press enter to kill the running thread : ")
kill_the_running_thread = True
# Calling main
main()

There's a problem when using globals as sentinels in conjunction with sleep. The issue is that the sleep may have only just started (5 minutes in OP's case) and so it could take nearly 5 minutes for the thread to realise that it should terminate.
A preferred (by me) technique is to use a queue. You can specify a timeout on a queue and, of course, it will respond almost immediately to any data passed to it. Here's an example:
from threading import Thread
from queue import Queue, Empty
def process(queue):
while True:
try:
queue.get(timeout=5)
break
except Empty as e:
pass
print('Doing work')
queue = Queue()
thread = Thread(target=process, args=(queue,))
thread.start()
input('Press enter to terminate the thread: ')
queue.put(None)
thread.join()
The process() function will block on the queue for up to 5 seconds (in this example). If there's nothing on the queue it will do its work. If there is something (we just pass None as the trigger), it will terminate immediately

You can try customizing the class, like this:
import datetime
import threading
import pymongo
import time
from pymongo import MongoClient
dbUrl = pymongo.MongoClient("mongodb://localhost:1245/")
dbName = dbUrl["student"]
dbCollectionName = dbName["student_course"]
class ThreadWithKill(threading.Thread):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._target = kwargs.get('target')
self._kill = threading.Event()
self._sleep_duration = 300 # 5 minutes
def run(self):
while True:
# If no kill signal is set, sleep for the duration of the interval.
# If kill signal comes in while sleeping, immediately wake up and handle
self._target() # execute passed function
is_killed = self._kill.wait(self._sleep_duration)
if is_killed:
break
def kill(self):
self._kill.set()
def doremon():
return "Hi Friends"
def insert_to_mongodb():
note_start_time = datetime.datetime.now()
msg = doremon()
note_end_time = datetime.datetime.now()
dt = {"message": msg, "start_time": note_start_time, "end_time": note_end_time}
rec_id1 = dbCollectionName.insert_one(dt)
def main():
my_thread = ThreadWithKill(target=insert_to_mongodb)
my_thread.start()
input("Press enter to kill the running thread : ")
my_thread.kill()
if __name__ == "__main__":
main()
This way there is no need for the kill_the_running_thread variable.
You'll need to test this yourself, because I don't have mongodb.

Python threads and linux ioctl wait

I have the following toy example for Python threading module
from __future__ import print_function
import threading
import time
import signal
import sys
import os
import time
class ThreadShutdown(Exception):
# Custom exception to allow clean thread exit
pass
def thread_shutdown(signum, frame):
print(" o Signal {} caught and raising ThreadShutdown exception".format(signum))
raise ThreadShutdown
def main():
"""
Register the signal handlers needed to stop
cleanly the child accessing thread
"""
signal.signal(signal.SIGTERM, thread_shutdown)
signal.signal(signal.SIGINT, thread_shutdown)
test_run_seconds = 120
try:
thread = ChildThread()
thread.start()
time.sleep(1)
while test_run_seconds > 0:
test_run_seconds -= 1
print(" o [{}] remaining time is {} seconds".format(time.asctime( time.localtime(time.time()) ), test_run_seconds))
time.sleep(1)
except ThreadShutdown:
thread.shutdown_flag.set()
thread.join()
print(" o ThreadShutdown procedure complete")
return
proc.terminate()
thread.shutdown_flag.set()
thread.join()
print(" o Test terminated")
class ChildThread(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.shutdown_flag = threading.Event()
def run(self):
while not self.shutdown_flag.is_set():
print(" o [{}] is the current time in child, sleep for 10s".format(time.asctime( time.localtime(time.time()))))
time.sleep(10)
return
if __name__ == "__main__":
sys.exit(main())
which behaves as expected (the main thread counts every second while the spawned thread prints only every 10 seconds.
I was trying to understand the behaviour of the same code snippet in presence of blocking waits in kernel mode in the spawned thread. For example, assume that the spawned thread now goes into a killable wait in an ioctl with a timeout of 10s, I would still expect to have the main thread counting every second. For some reason, it instead counts every 10s, as if it was blocked as well in the wait of the spawned thread. What is the reason?

How to correctly handle SIGINT to close files/connections

I want to implement a proper SIGINT handling in my script, which opens multiple files and a database connection. These should be closed if the script is CTRL+C'd or somehow else interrupted.
Previously I've used the KeyboardInterrupt exception to catch CTRL+C, there I checked if files/connections are defined, if so close them, then exit.
Is this really the pythonic way to do it, or is it better adviced to use signal handlers? e.g.
import signal, sys, time
def handler(signum, frame):
print("..kthxbye")
sys.exit(1)
def main():
signal.signal(signal.SIGINT, handler)
i = 0
while True:
print(i)
i += 1
time.sleep(1)
if __name__ == "__main__":
main()
This seems cleaner to me, yet don't I know how I would pass filenames or database connections to the handler.

I would rather catch the KeyboardInterrupt exception on the main thread. KeyboardInterrupt is the result of the default SIGINT handler of python. The exception handler of a KeyboardInterrupt exception is a much safer/friendly context than what you are in when catching SIGINT directly.
if __name__ == '__main__':
try:
main()
except KeyboardInterrupt:
cleanup()
EDIT: Here is how to share variables (state) between the two methods:
Procedural:
import sys, time
class SharedState:
def __init__(self):
self.var0 = 42
self.var1 = 'string'
# method 1
shared_variable = 'woof woof'
# method 2: avoiding global declarations in functions
shared_state = SharedState()
def main():
# In order to write a global variable you need a global
# declaration otherwise the assignment would create a
# local variable
global shared_variable
shared_variable = 5
shared_state.var0 = 10
time.sleep(10)
def cleanup():
print shared_variable
print shared_state.var0
sys.exit(1)
if __name__ == '__main__':
try:
main()
except KeyboardInterrupt:
cleanup()
Object oriented (my preference):
import sys, time
# method 3: object oriented programming
class Program:
def __init__(self):
self.var0 = 42
self.var1 = 'string'
def main(self):
self.var0 = 5
self.var1 = 'woof woof'
time.sleep(10)
def cleanup(self):
# both main and cleanup can access the member
# variables of this class
print self.var0
print self.var1
sys.exit(1)
def execute(self):
try:
self.main()
except KeyboardInterrupt:
self.cleanup()
if __name__ == '__main__':
Program().execute()

My suggestion is to use the signal library to handle the signals. Signals are not exceptions and they are part of Inter Process Communication (IPC) infrastructure of the Operating System.
Signals can help you to communicate with your program, like reloading the configuration file, closing your log file handler during log rotation and so on. Most of the daemon process like apache dose it.
Shell scripts have trap command to process the signals and take appropriate actions based on the signals captured.
Generally python closes all file handlers and database connection automatically during the time of exit. But to be safe we can have a function to handle them implicitly.
Below code traps SIGINT and closes the files properly.
import signal
import sys
die = False
def handler(signum, frame):
global die
print('Got SIGINT.')
die = True
def closeFile(fh):
fh.flush()
fh.close()
signal.signal(signal.SIGINT, handler)
fh = open('/tmp/a.txt', 'w')
while True:
data = input('> ')
if data == 'q':
closeFile(fh)
break
else:
fh.write(data + '\n')
if die:
closeFile(fh)
print('Completed cleanup.. ')
sys.exit()

python asyncronous thread exception handling

I'm trying to implement a timeout functionality in Python.
It works by wrapping functions with a function decorator that calls the function as a thread but also calls a 'watchdog' thread that will raise an exception in the function thread after a specified period has elapsed.
It currently works for threads that don't sleep. During the do_rand call, I suspect the 'asynchronous' exception is actually being called after the time.sleep call and after the execution has moved beyond the try/except block, as this would explain the Unhandled exception in thread started by error. Additionally, the error from the do_rand call is generated 7 seconds after the call (the duration of time.sleep).
How would I go about 'waking' a thread up (using ctypes?) to get it to respond to an asynchronous exception ?
Or possibly a different approach altogether ?
Code:
# Import System libraries
import ctypes
import random
import sys
import threading
import time
class TimeoutException(Exception):
pass
def terminate_thread(thread, exc_type = SystemExit):
"""Terminates a python thread from another thread.
:param thread: a threading.Thread instance
"""
if not thread.isAlive():
return
exc = ctypes.py_object(exc_type)
res = ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(thread.ident), exc)
if res == 0:
raise ValueError("nonexistent thread id")
elif res > 1:
# """if it returns a number greater than one, you're in trouble,
# and you should call it again with exc=NULL to revert the effect"""
ctypes.pythonapi.PyThreadState_SetAsyncExc(thread.ident, None)
raise SystemError("PyThreadState_SetAsyncExc failed")
class timeout_thread(threading.Thread):
def __init__(self, interval, target_thread):
super(timeout_thread, self).__init__()
self.interval = interval
self.target_thread = target_thread
self.done_event = threading.Event()
self.done_event.clear()
def run(self):
timeout = not self.done_event.wait(self.interval)
if timeout:
terminate_thread(self.target_thread, TimeoutException)
class timeout_wrapper(object):
def __init__(self, interval = 300):
self.interval = interval
def __call__(self, f):
def wrap_func(*args, **kwargs):
thread = threading.Thread(target = f, args = args, kwargs = kwargs)
thread.setDaemon(True)
timeout_ticker = timeout_thread(self.interval, thread)
timeout_ticker.setDaemon(True)
timeout_ticker.start()
thread.start()
thread.join()
timeout_ticker.done_event.set()
return wrap_func
#timeout_wrapper(2)
def print_guvnah():
try:
while True:
print "guvnah"
except TimeoutException:
print "blimey"
def print_hello():
try:
while True:
print "hello"
except TimeoutException:
print "Whoops, looks like I timed out"
def do_rand(*args):
try:
rand_num = 7 #random.randint(0, 10)
rand_pause = 7 #random.randint(0, 5)
print "Got rand: %d" % rand_num
print "Waiting for %d seconds" % rand_pause
time.sleep(rand_pause)
except TimeoutException:
print "Waited too long"
print_guvnah()
timeout_wrapper(3)(print_hello)()
timeout_wrapper(2)(do_rand)()

The problem is that time.sleep blocks. And it blocks really hard, so the only thing that can actually interrupt it is process signals. But the code with it gets really messy and in some cases even signals don't work ( when for example you are doing blocking socket.recv(), see this: recv() is not interrupted by a signal in multithreaded environment ).
So generally interrupting a thread (without killing entire process) cannot be done (not to mention that someone can simply override your signal handling from a thread).
But in this particular case instead of using time.sleep you can use Event class from threading module:
Thread 1
from threading import Event
ev = Event()
ev.clear()
state = ev.wait(rand_pause) # this blocks until timeout or .set() call
Thread 2 (make sure it has access to the same ev instance)
ev.set() # this will unlock .wait above
Note that state will be the internal state of the event. Thus state == True will mean that it was unlocked with .set() while state == False will mean that timeout occured.
Read more about events here:
http://docs.python.org/2/library/threading.html#event-objects

You'd need to use something other than sleep, or you'd need to send a signal to the other thread in order to make it wake up.
One option I've used is to set up a pair of file descriptors and use select or poll instead of sleep, this lets you write something to the file descriptor to wake up the other thread. Alternatively you just wear waiting until the sleep finishes if all you need is for the operation to error out because it took too long and nothing else is depending on it.

Python: Catch Ctrl-C command. Prompt "really want to quit (y/n)", resume execution if no

I have a program that may have a lengthy execution. In the main module I have the following:
import signal
def run_program()
...time consuming execution...
def Exit_gracefully(signal, frame):
... log exiting information ...
... close any open files ...
sys.exit(0)
if __name__ == '__main__':
signal.signal(signal.SIGINT, Exit_gracefully)
run_program()
This works fine, but I'd like the possibility to pause execution upon catching SIGINT, prompting the user if they would really like to quit, and resuming where I left off in run_program() if they decide they don't want to quit.
The only way I can think of doing this is running the program in a separate thread, keeping the main thread waiting on it and ready to catch SIGINT. If the user wants to quit the main thread can do cleanup and kill the child thread.
Is there a simpler way?

The python signal handlers do not seem to be real signal handlers; that is they happen after the fact, in the normal flow and after the C handler has already returned. Thus you'd try to put your quit logic within the signal handler. As the signal handler runs in the main thread, it will block execution there too.
Something like this seems to work nicely.
import signal
import time
import sys
def run_program():
while True:
time.sleep(1)
print("a")
def exit_gracefully(signum, frame):
# restore the original signal handler as otherwise evil things will happen
# in raw_input when CTRL+C is pressed, and our signal handler is not re-entrant
signal.signal(signal.SIGINT, original_sigint)
try:
if raw_input("\nReally quit? (y/n)> ").lower().startswith('y'):
sys.exit(1)
except KeyboardInterrupt:
print("Ok ok, quitting")
sys.exit(1)
# restore the exit gracefully handler here
signal.signal(signal.SIGINT, exit_gracefully)
if __name__ == '__main__':
# store the original SIGINT handler
original_sigint = signal.getsignal(signal.SIGINT)
signal.signal(signal.SIGINT, exit_gracefully)
run_program()
The code restores the original signal handler for the duration of raw_input; raw_input itself is not re-entrable, and re-entering it
will lead to RuntimeError: can't re-enter readline being raised from time.sleep which is something we don't want as it is harder to catch than KeyboardInterrupt. Rather, we let 2 consecutive Ctrl-C's to raise KeyboardInterrupt.

from https://gist.github.com/rtfpessoa/e3b1fe0bbfcd8ac853bf
#!/usr/bin/env python
import signal
import sys
def signal_handler(signal, frame):
# your code here
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
Bye!

when procedure end then do something
suppose you just want to the procedure will do something after the task end
import time
class TestTask:
def __init__(self, msg: str):
self.msg = msg
def __enter__(self):
print(f'Task Start!:{self.msg}')
return self
def __exit__(self, exc_type, exc_val, exc_tb):
print('Task End!')
#staticmethod
def do_something():
try:
time.sleep(5)
except:
pass
with TestTask('Hello World') as task:
task.do_something()
when the process leaves with that will run __exit__ even with KeyboardInterrupt happen that are same.
if you don't like to see the error, add try ... except ...
#staticmethod
def do_something():
try:
time.sleep(5)
except:
pass
pause, continue, reset, and etc.
I don't have a perfect solution, but it may be useful to you.
It's means divided your process to many subprocesses and save it that finished.it will not be executed again since you find it already done.
import time
from enum import Enum
class Action(Enum):
EXIT = 0
CONTINUE = 1
RESET = 2
class TestTask:
def __init__(self, msg: str):
self.msg = msg
def __enter__(self):
print(f'Task Start!:{self.msg}')
return self
def __exit__(self, exc_type, exc_val, exc_tb):
print('Task End!')
def do_something(self):
tuple_job = (self._foo, self._bar) # implement by yourself
list_job_state = [0] * len(tuple_job)
dict_keep = {} # If there is a need to communicate between jobs, and you don’t want to use class members, you can use this method.
while 1:
try:
for idx, cur_process in enumerate(tuple_job):
if not list_job_state[idx]:
cur_process(dict_keep)
list_job_state[idx] = True
if all(list_job_state):
print('100%')
break
except KeyboardInterrupt:
print('KeyboardInterrupt. input action:')
msg = '\n\t'.join([f"{action + ':':<10}{str(act_number)}" for act_number, action in
enumerate([name for name in vars(Action) if not name.startswith('_')])
])
case = Action(int(input(f'\t{msg}\n:')))
if case == Action.EXIT:
break
if case == Action.RESET:
list_job_state = [0] * len(tuple_job)
#staticmethod
def _foo(keep_dict: dict) -> bool: # implement by yourself
time.sleep(2)
print('1%')
print('2%')
print('...')
print('60%')
keep_dict['status_1'] = 'status_1'
return True
#staticmethod
def _bar(keep_dict: dict) -> bool: # implement by yourself
time.sleep(2)
print('61%')
print(keep_dict.get('status_1'))
print('...')
print('99%')
return True
with TestTask('Hello World') as task:
task.do_something()
console
input action number:2
Task Start!:Hello World
1%
2%
...
60%
KeyboardInterrupt. input action:
EXIT: 0
CONTINUE: 1
RESET: 2
:1
61%
status_1
...
99%
100%
Task End!