Ok, I've wrote this class based in a bunch of others Spinner classes that I've googled in Google Code Search.
It's working as intended, but I'm looking for a better way to handle KeyboardInterrupt and SystemExit exceptions. Is there better approaches?
Here's my code:
#!/usr/bin/env python
import itertools
import sys
import threading
class Spinner(threading.Thread):
'''Represent a random work indicator, handled in a separate thread'''
# Spinner glyphs
glyphs = ('|', '/', '-', '\\', '|', '/', '-')
# Output string format
output_format = '%-78s%-2s'
# Message to output while spin
spin_message = ''
# Message to output when done
done_message = ''
# Time between spins
spin_delay = 0.1
def __init__(self, *args, **kwargs):
'''Spinner constructor'''
threading.Thread.__init__(self, *args, **kwargs)
self.daemon = True
self.__started = False
self.__stopped = False
self.__glyphs = itertools.cycle(iter(self.glyphs))
def __call__(self, func, *args, **kwargs):
'''Convenient way to run a routine with a spinner'''
self.init()
skipped = False
try:
return func(*args, **kwargs)
except (KeyboardInterrupt, SystemExit):
skipped = True
finally:
self.stop(skipped)
def init(self):
'''Shows a spinner'''
self.__started = True
self.start()
def run(self):
'''Spins the spinner while do some task'''
while not self.__stopped:
self.spin()
def spin(self):
'''Spins the spinner'''
if not self.__started:
raise NotStarted('You must call init() first before using spin()')
if sys.stdin.isatty():
sys.stdout.write('\r')
sys.stdout.write(self.output_format % (self.spin_message,
self.__glyphs.next()))
sys.stdout.flush()
time.sleep(self.spin_delay)
def stop(self, skipped=None):
'''Stops the spinner'''
if not self.__started:
raise NotStarted('You must call init() first before using stop()')
self.__stopped = True
self.__started = False
if sys.stdin.isatty() and not skipped:
sys.stdout.write('\b%s%s\n' % ('\b' * len(self.done_message),
self.done_message))
sys.stdout.flush()
class NotStarted(Exception):
'''Spinner not started exception'''
pass
if __name__ == '__main__':
import time
# Normal example
spinner1 = Spinner()
spinner1.spin_message = 'Scanning...'
spinner1.done_message = 'DONE'
spinner1.init()
skipped = False
try:
time.sleep(5)
except (KeyboardInterrupt, SystemExit):
skipped = True
finally:
spinner1.stop(skipped)
# Callable example
spinner2 = Spinner()
spinner2.spin_message = 'Scanning...'
spinner2.done_message = 'DONE'
spinner2(time.sleep, 5)
Thank you in advance.
You probably don't need to worry about catching SystemExit as it is raised by sys.exit(). You might want to catch it to clean up some resources just before your program exits.
The other way to catch KeyboardInterrupt is to register a signal handler to catch SIGINT. However for your example using try..except makes more sense, so you're on the right track.
A few minor suggestions:
Perhaps rename the __call__ method to start, to make it more clear you're starting a job.
You might also want to make the Spinner class reusable by attaching a new thread within the start method, rather than in the constructor.
Also consider what happens when the user hits CTRL-C for the current spinner job -- can the next job be started, or should the app just exit?
You could also make the spin_message the first argument to start to associate it with the task about to be run.
For example, here is how someone might use Spinner:
dbproc = MyDatabaseProc()
spinner = Spinner()
spinner.done_message = 'OK'
try:
spinner.start("Dropping the database", dbproc.drop, "mydb")
spinner.start("Re-creating the database", dbproc.create, "mydb")
spinner.start("Inserting data into tables", dbproc.populate)
...
except (KeyboardInterrupt, SystemExit):
# stop the currently executing job
spinner.stop()
# do some cleanup if needed..
dbproc.cleanup()
Related
I wanted to create a custom Thread class that is able to propagate an exception it comes across to the main thread. My implementation is as follows:
class VerseThread(threading.Thread):
def __init__(self, args):
super().__init__(self, args=args)
# self.scraper = scraper
def run(self):
self.exc = None
try:
book, abbrev, template, chapter = self.args
self.parser.parse(book, abbrev, template, chapter)
except ChapterNotFoundError as e:
self.exc = e
def join(self):
threading.Thread.join(self)
if self.exc:
raise self.exc
This is supposed to run in the following method, inside a Scraper class (it's all inside a ẁhile true):
for book, abbrev, testament in self.books[init:end]:
base_chapter = 1
while True:
threads = []
if testament == 'ot':
for i in range(3):
threads.append(VerseThread(args=(book, abbrev, OT_TEMPLATE, base_chapter+i)))
else:
for i in range(3):
threads.append(VerseThread(args=(book, abbrev, NT_TEMPLATE, base_chapter+i)))
try:
for thread in threads:
if not thread.is_alive():
thread.start()
for thread in threads:
thread.join()
base_chapter += 3
except ChapterNotFoundError as e:
LOGGER.info(f"{{PROCESS {multiprocessing.current_process().pid}}} - Chapter {e.chapter} not found in {book}, exiting book...")
break
The issue is, if I run it like presented here, I get the error assert group is None, "group argument must be None for now". However, when I run it using Thread(target=self.parse, args=(book, abbrev, OT_TEMPLATE, base_chapter+1)) instead of VerseThread(args=(book, abbrev, OT_TEMPLATE, base_chapter+i)), it works just fine, but the exception is of course still there. What's wrong with my code? How can I get rid of this error?
EDIT: Upon further testing, it seems that what I'm trying to do works fine when I use thread.run() instead of thread.start(), but then only one thread is being used, which is a problem. This, however, means that the error must be in the start() method, but I've no idea what to do.
You have several errors. First, if you are using super() as in super().__init__(self, target=target, args=args), you do not pass self explicitly as an argument. Second, to handle any possible thread-initializer arguments, your signature for this method should just be as follows:
class VerseThread(threading.Thread):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
... # rest of the code omitted
But since your __init__ method does not do anything but call the parent's __init__ method with any passed arguments, there is now no need to even override this method.
Finally, the attributes that you are interested in are not args but rather _args and _kwargs (in case keyword arguments are specified). Also, you have specified self.parser, but I do not see where that attribute has been set.
import threading
class ChapterNotFoundError(Exception):
pass
class VerseThread(threading.Thread):
def run(self):
self.exc = None
try:
book, abbrev, template, chapter = self._args
self.parser.parse(book, abbrev, template, chapter)
except ChapterNotFoundError as e:
self.exc = e
def join(self):
threading.Thread.join(self) # Or: super().join()
if self.exc:
raise self.exc
for book, abbrev, testament in self.books[init:end]:
base_chapter = 1
while True:
threads = []
if testament == 'ot':
for i in range(3):
threads.append(VerseThread(args=(book, abbrev, OT_TEMPLATE, base_chapter+i)))
else:
for i in range(3):
threads.append(VerseThread(args=(book, abbrev, NT_TEMPLATE, base_chapter+i)))
try:
for thread in threads:
if not thread.is_alive():
thread.start()
for thread in threads:
thread.join()
base_chapter += 3
except ChapterNotFoundError as e:
LOGGER.info(f"{{PROCESS {multiprocessing.current_process().pid}}} - Chapter {e.chapter} not found in {book}, exiting book...")
break
Improvement
Accessing quasi-private attributes, such as self._args is a potentially dangerous thing and should be avoided.
I can see the value of creating a subclass of Thread that will catch exceptions in the "worker" function it is to execute and then "propogate" it back to the main thread when it joins the thread. But I believe such a class should be general purpose and work with any type of worker function. In general, I don't like to have application-specific code (business logic) in a multithreading.Thread or multiprocessing.Pool subclass. I instead prefer having my business logic coded within a function or class method(s) that can then be used in multithreading, multiprocessing or serial processing as you see fit. The following is how I would code the Thread subclass (I have named it PropogateExceptionThread, but chose whatever name you wish) and I might use it:
import threading
class PropogateExceptionThread(threading.Thread):
def run(self):
self.exc = None
try:
super().run()
except Exception as e:
self.exc = e
def join(self):
super().join()
if self.exc:
raise self.exc
def worker(x):
if x < 10 or x > 20:
raise ValueError(f'Bad value for argument x = {x}')
t = PropogateExceptionThread(target=worker, args=(1,))
t.start()
try:
t.join()
except Exception as e:
print('The thread raised an exception:', e)
Prints:
The thread raised an exception: Bad value for argument x = 1
When I hit ctrl-c the prompt pops up on the screen and I input no
because I want to keep running the program but the program exits anyway.
Where am I going wrong? Is this functionality even possible?
from functools import wraps
import sys
class CleanExit(object):
def __init__(self, *args, **kw):
# You can supply an optional function
# to be called when the KeyboardInterrupt
# takes place.
# If the function doesn't require any arguments:
# #CleanExit(handler=func)
# If the function requires arguments:
# #CleanExit('foo', handler=handle)
self.kw = kw
self.args = args
self.handler = kw.get('handler')
def __call__(self, original_func):
decorator_self = self
#wraps(original_func)
def wrappee(*args, **kwargs):
try:
original_func(*args,**kwargs)
except KeyboardInterrupt:
if self.handler:
self.handler(*self.args)
else:
sys.exit(0)
return wrappee
def handle():
answer = raw_input("Are you sure you want to exit?").lower().strip()
if 'y' in answer:
sys.exit(0)
#CleanExit(handler=handle)
def f():
while 1: pass
f()
Your problem is that you're not doing anything to continue the function after handling it - so your code handles the interrupt, and then exits anyway. You can recursively re-enter wrappee if the handler exits successfully like this:
def __call__(self, original_func):
decorator_self = self
#wraps(original_func)
def wrappee(*args, **kwargs):
try:
original_func(*args,**kwargs)
except KeyboardInterrupt:
if self.handler:
self.handler(*self.args)
wrappee(*args, **kwargs)
else:
sys.exit(0)
return wrappee
Now this should work. Note that this is a little bit naughty, as Python can't optimise tail calls, so if you KeyboardInterrupt more often than sys.getrecursionlimit(), Python will run out of stack frames and crash.
EDIT: That was silly - having thought about it, this function is so trivial to derecurse by hand it probably doesn't even count.
def __call__(self, original_func):
decorator_self = self
#wraps(original_func)
def wrappee(*args, **kwargs):
while True:
try:
original_func(*args,**kwargs)
except KeyboardInterrupt:
if self.handler:
self.handler(*self.args)
else:
sys.exit(0)
return wrappee
should also work just fine.
Below there is some fully functioning code.
I am planning to execute this code through command line, however I would like it to end after 60 seconds.
Does anyone know the best way of going about this?
Thanks in advance.
import time
class listener(StreamListener):
def on_data(self, data):
try:
print data
saveFile = open('twitDB.csv','a')
saveFile.write(data)
saveFile.write('\n')
saveFile.close()
return True
except BaseException, e:
print 'failed ondata,' ,str(e)
time.sleep(5)
def on_error(self, status):
print status
Try this out:
import os
import time
from datetime import datetime
from threading import Timer
def exitfunc():
print "Exit Time", datetime.now()
os._exit(0)
Timer(5, exitfunc).start() # exit in 5 seconds
while True: # infinite loop, replace it with your code that you want to interrupt
print "Current Time", datetime.now()
time.sleep(1)
There are some more examples in this StackOverflow question: Executing periodic actions in Python
I think the use of os._exit(0) is discouraged, but I'm not sure. Something about this doesn't feel kosher. It works, though.
You could move your code into a daemon thread and exit the main thread after 60 seconds:
#!/usr/bin/env python
import time
import threading
def listen():
print("put your code here")
t = threading.Thread(target=listen)
t.daemon = True
t.start()
time.sleep(60)
# main thread exits here. Daemon threads do not survive.
Use signal.ALARM to get notified after a specified time.
import signal, os
def handler(signum, frame):
print '60 seconds passed, exiting'
cleanup_and_exit_your_code()
# Set the signal handler and a 60-second alarm
signal.signal(signal.SIGALRM, handler)
signal.alarm(60)
run_your_code()
From your example it is not obvious what the code will exactly do, how it will run and what kind of loop it will iterate. But you can easily implement the ALARM signal to get notified after the timeout has expired.
This is my favorite way of doing timeout.
def timeout(func, args=None, kwargs=None, TIMEOUT=10, default=None, err=.05):
if args is None:
args = []
elif hasattr(args, "__iter__") and not isinstance(args, basestring):
args = args
else:
args = [args]
kwargs = {} if kwargs is None else kwargs
import threading
class InterruptableThread(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.result = None
def run(self):
try:
self.result = func(*args, **kwargs)
except:
self.result = default
it = InterruptableThread()
it.start()
it.join(TIMEOUT* (1 + err))
if it.isAlive():
return default
else:
return it.result
I hope this is an easy way to execute a function periodically and end after 60 seconds:
import time
import os
i = 0
def executeSomething():
global i
print(i)
i += 1
time.sleep(1)
if i == 10:
print('End')
os._exit(0)
while True:
executeSomething()
One of the modules in an app I'm working on is intended to be used as a long-running process on Linux, and I would like it to gracefully handle SIGTERM, SIGHUP and possibly other signals. The core part of the program is in fact a loop which periodically runs a function (which in turn wakes up another thread, but this is less important). It looks more or less like this:
while True:
try:
do_something()
sleep(60)
except KeyboardInterrupt:
break
cleanup_and_exit()
What I'd like to add now is to catch SIGTERM and exit the loop, the same way a KeyboardInterrupt exception would.
One thought I have is to add a flag which will be set to True by the signal handler function, and replace the sleep(60) with sleep(0.1) or whatever, with a counter that counts seconds:
_exit_flag = False
while not _exit_flag:
try:
for _ in xrange(600):
if _exit_flag: break
do_something()
sleep(0.1)
except KeyboardInterrupt:
break
cleanup_and_exit()
and somewhere else:
def signal_handler(sig, frame):
_exit_flag = True
But I'm not sure this is the best / most efficient way to do it.
Rather than using a sentinel in the main loop and thus having to wake more frequently than you'd really like to check for it, why not push the cleanup in to the handler? Something like:
class BlockingAction(object):
def __new__(cls, action):
if isinstance(action, BlockingAction):
return action
else:
new_action = super(BlockingAction, cls).__new__(cls)
new_action.action = action
new_action.active = False
return new_action
def __call__(self, *args, **kwargs):
self.active = True
result = self.action(*args, **kwargs)
self.active = False
return result
class SignalHandler(object):
def __new__(cls, sig, action):
if isinstance(action, SignalHandler):
handler = action
else:
handler = super(SignalHandler, cls).__new__(cls)
handler.action = action
handler.blocking_actions = []
signal.signal(sig, handler)
return handler
def __call__(self, signum, frame):
while any(a.active for a in self.blocking_actions):
time.sleep(.01)
return self.action()
def blocks_on(self, action):
blocking_action = BlockingAction(action)
self.blocking_actions.append(blocking_action)
return blocking_action
def handles(signal):
def get_handler(action):
return SignalHandler(signal, action)
return get_handler
#handles(signal.SIGTERM)
#handles(signal.SIGHUP)
#handles(signal.SIGINT)
def cleanup_and_exit():
# Note that this assumes that this method actually exits the program explicitly
# If it does not, you'll need some form of sentinel for the while loop
pass
#cleanup_and_exit.blocks_on
def do_something():
pass
while True:
do_something()
time.sleep(60)
I'm wondering if the following class is sound. I'm using it to launch a bunch of simulators for each test in my test environment.
class SubProcessInOwnThread(threading.Thread):
def __init__(self, arguments, currentWorkingDirectory):
self.arguments = arguments
self.currentWorkingDirectory = currentWorkingDirectory
threading.Thread.__init__(self)
self.isTerminated = False
def run(self):
try:
self.subProcess = subprocess.Popen(self.arguments, cwd=self.currentWorkingDirectory)
self.subProcess.wait()
finally:
self.isTerminated = True
def kill(self):
while not self.isTerminated:
try:
self.subProcess.kill()
except:
time.sleep(0.1)
Some senarios:
# Normal
subProcessThreadArguments = ["cmd.exe"]
subProcessThread = SubProcessInOwnThread(subProcessThreadArguments,r"C:\\")
subProcessThread.start()
time.sleep(5)
subProcessThread.kill()
# Process killed very quickly
subProcessThreadArguments = ["cmd.exe"]
subProcessThread = SubProcessInOwnThread(subProcessThreadArguments,r"C:\\")
subProcessThread.start()
subProcessThread.kill()
# Incorrect configuration
subProcessThreadArguments = ["cmdsfgfg.exe"]
subProcessThread = SubProcessInOwnThread(subProcessThreadArguments,r"C:\\")
subProcessThread.start()
time.sleep(5)
subProcessThread.kill()
So I can create simulators like this:
subProcessThreadArguments1 = ["sim1.exe"]
subProcessThread1 = SubProcessInOwnThread(subProcessThreadArguments1,r"C:\\")
subProcessThread1.start()
subProcessThreadArguments2 = ["sim2.exe"]
subProcessThread2 = SubProcessInOwnThread(subProcessThreadArguments2,r"C:\\")
subProcessThread2.start()
# do test...
subProcessThread1.kill()
subProcessThread2.kill()
I'd be interested in any improvents. Should I consider the use of the with keyword? If so, what would the benifits be?
Thanks!
I don't see the point of having a separate thread being stuck in wait() here. Working directly on the subprocess would work like
class SubProcessWithoutThread(object):
def __init__(self, arguments, currentWorkingDirectory):
self.arguments = arguments
self.currentWorkingDirectory = currentWorkingDirectory
self.isTerminated = False
def start(self):
self.subProcess = subprocess.Popen(self.arguments, cwd=self.currentWorkingDirectory)
def kill(self):
while self.subProcess.poll() is None:
try:
self.subProcess.kill()
except:
time.sleep(0.1)
__enter__ = start
def __exit__(self, *x):
self.kill()
(untested)
I have added the methods for a context manager, but I cannot see how that would help you as it would be quite a bunch of with statements which you would have to create, including the necessary indentation.
But maybe I have got your intention wrong...