I want to create one program in which two lists of hosts are available. I want to read data from each host. It will take around 5-10 seconds so I want to read each host data with different thread.
I created below code and it is working as per my expectations but only problem is when I'm pressing Ctrl+c, program didn't terminate.
My code:
import threading
import time,os,sys
import signal
is_running = True
def signal_handler(signal, frame):
print "cleaning up...please wait..."
v1.stop()
v2.stop()
global is_running
is_running = False
class Thread2(threading.Thread):
def __init__(self, function,args):
self.running = False
self.function = function
self.args = args
super(Thread2, self).__init__()
def start(self):
self.running = True
super(Thread2, self).start()
def run(self):
while is_running:
self.function(self.args)
time.sleep(time_interval)
def stop(self):
self.running = False
def b_iterate(hostnames):
for host_name in hostnames:
v = Thread2(function = read_cet_data,args = host_name)
v.start()
def read_b_data(host):
#
#reading some data from current host (5-10 seconds processing)
#
#here, this thread is not neccessary, want to stop or kill or terminate it
if threading.current_thread().isAlive():
threading.current_thread().stop()
def a_iterate(entp_hostnames):
for host_name in entp_hostnames:
v = Thread2(function = read_entp_data,args = host_name)
v.start()
def read_a_data(host):
#
#reading some data from current host (5-10 seconds processing)
#
#here, this thread is not neccessary, want to stop or kill or terminate it
if threading.current_thread().isAlive():
threading.current_thread().stop()
if __name__ == "__main__":
signal.signal(signal.SIGINT, signal_handler)
#a_hostnmaes & b_hostnmaes are the lists of hostnames
v1 = Thread2(function = a_iterate,args = a_hostnames)
v2 = Thread2(function = b_iterate,args = b_hostnames)
v1.start()
v2.start()
while is_running:
pass
How I can make this program terminate after pressing Ctrl+c. Am I missing something?
If you just want control C to finish everything, there is no need to use a stop function in threads. You can just daemonise them:
v1 = Thread2(function = a_iterate,args = a_hostnames)
v2 = Thread2(function = b_iterate,args = b_hostnames)
v1.daemon = True
v2.daemon = True
v1.start()
v2.start()
As soon as your main program dies, these threads die as well. You need to add .daemon = True to all other locations in the code where a thread is created.
Hannu
You can either
catch KeyboardInterrupt in main thread
set a flag so another threads can detect it and exit
or
catch KeyboardInterrupt
call os._exit()
Related
Hey I'm learning psutil package and I want to know how to display current CPU usage when function is in progress? I suppose I need some threading or something like this, but how to do it? Thank u for any answers.
import psutil
import random
def iHateThis():
tab = []
for i in range(100000):
tab.append(random.randint(1, 10000))
tab.sort()
return tab;
while(True):
currentProcess = psutil.Process()
print(currentProcess.cpu_percent(interval=1))
You can use threading to run iHateThis or to run function with cpu_percent(). I choose second version. I will run cpu_percent() in thread.
Because it uses while True so thread would run forever and there wouldn't be nice method to stop thread so I use global variaable running with while running to have method to stop this loop.
import threading
import psutil
def display_cpu():
global running
running = True
currentProcess = psutil.Process()
# start loop
while running:
print(currentProcess.cpu_percent(interval=1))
def start():
global t
# create thread and start it
t = threading.Thread(target=display_cpu)
t.start()
def stop():
global running
global t
# use `running` to stop loop in thread so thread will end
running = False
# wait for thread's end
t.join()
and now I can use it to start and stop thread which will display CPU. Because I may have to stop process using Ctrl+C so it will raise error so I use try/finally to stop thread even if there will be error.
def i_hate_this():
tab = []
for i in range(1000000):
tab.append(random.randint(1, 10000))
tab.sort()
return tab
# ---
start()
try:
result = i_hate_this()
finally: # stop thread even if I press Ctrl+C
stop()
Full code:
import random
import threading
import psutil
def display_cpu():
global running
running = True
currentProcess = psutil.Process()
# start loop
while running:
print(currentProcess.cpu_percent(interval=1))
def start():
global t
# create thread and start it
t = threading.Thread(target=display_cpu)
t.start()
def stop():
global running
global t
# use `running` to stop loop in thread so thread will end
running = False
# wait for thread's end
t.join()
# ---
def i_hate_this():
tab = []
for i in range(1000000):
tab.append(random.randint(1, 10000))
tab.sort()
return tab
# ---
start()
try:
result = i_hate_this()
finally: # stop thread even if I press Ctrl+C
stop()
BTW: this can be converted to class which inherits from class Thread and then it can hide variable running in class.
import psutil
import random
import threading
class DisplayCPU(threading.Thread):
def run(self):
self.running = True
currentProcess = psutil.Process()
while self.running:
print(currentProcess.cpu_percent(interval=1))
def stop(self):
self.running = False
# ----
def i_hate_this():
tab = []
for i in range(1000000):
tab.append(random.randint(1, 10000))
tab.sort()
return tab
# ---
display_cpu = DisplayCPU()
display_cpu.start()
try:
result = i_hate_this()
finally: # stop thread even when I press Ctrl+C
display_cpu.stop()
It could be also converted to context manager to run it as
with display_cpu():
i_hate_this()
but I skip this part.
You can do this with the multiprocessing library. multiprocessing.Process is a class that represents a threaded process, is initiated with a function and name, and can be run at any time with .start().
import multiprocessing
import psutil
import random
def iHateThis():
tab = []
for i in range(100000):
tab.append(random.randint(1, 10000))
tab.sort()
return tab;
hate = multiprocessing.Process(name='hate', target=iHateThis)
hate.start()
while(True):
currentProcess = psutil.Process()
print(currentProcess.cpu_percent(interval=1))
I don't think you need to use psutil Process class as I think it is intended to be used to monitor a specific process. Using the code snippet from #furas (the accepted answer), you can do it with a thread like this:
def run(self):
self.run = True
while self.run:
psutil.cpu_percent(interval=1)
it works the same as the accepted answer in the following case:
_monitor.start()
try:
for i in range(50):
time.sleep(0.2)
finally:
_monitor.stop()
If you don't want to code it, I am doing it in a public repo if it can be of any help for someone: https://github.com/GTimothee/monitor
Let approach the problem differently and propose a decorator that can serve to measure CPU utilization while running
from functools import partial, wraps
def log_cpu_usage(func=None, msg_prefix: str = None):
"""
This function is a decorator that measures the execution time of a function and logs it.
"""
debug = True
if not debug:
return func
if func is None:
return partial(log_cpu_usage, msg_prefix=msg_prefix)
def new_func(data: mp.Queue, *args, **kwargs):
result = func(*args, **kwargs)
data.put(result)
#wraps(func)
def trace_execution(*args, **kwargs):
manager = mp.Queue() # to save return val between multi process
worker_process = mp.Process(target=new_func, args=(manager, *args), kwargs=kwargs)
worker_process.start()
p = psutil.Process(worker_process.pid)
cpu_percents = []
while worker_process.is_alive(): # while the subprocess is running
cpu_percents.append(p.cpu_percent() / psutil.cpu_count())
time.sleep(0.01)
worker_process.join()
ret_values = manager.get()
return sum(cpu_percents) / len(cpu_percents), ret_values
#log_cpu_usage
def iHateThis():
pass
How can I start and stop a thread with my poor thread class?
It is in loop, and I want to restart it again at the beginning of the code. How can I do start-stop-restart-stop-restart?
My class:
import threading
class Concur(threading.Thread):
def __init__(self):
self.stopped = False
threading.Thread.__init__(self)
def run(self):
i = 0
while not self.stopped:
time.sleep(1)
i = i + 1
In the main code, I want:
inst = Concur()
while conditon:
inst.start()
# After some operation
inst.stop()
# Some other operation
You can't actually stop and then restart a thread since you can't call its start() method again after its run() method has terminated. However you can make one pause and then later resume its execution by using a threading.Condition variable to avoid concurrency problems when checking or changing its running state.
threading.Condition objects have an associated threading.Lock object and methods to wait for it to be released and will notify any waiting threads when that occurs. Here's an example derived from the code in your question which shows this being done. In the example code I've made the Condition variable a part of Thread subclass instances to better encapsulate the implementation and avoid needing to introduce additional global variables:
from __future__ import print_function
import threading
import time
class Concur(threading.Thread):
def __init__(self):
super(Concur, self).__init__()
self.iterations = 0
self.daemon = True # Allow main to exit even if still running.
self.paused = True # Start out paused.
self.state = threading.Condition()
def run(self):
self.resume()
while True:
with self.state:
if self.paused:
self.state.wait() # Block execution until notified.
# Do stuff...
time.sleep(.1)
self.iterations += 1
def pause(self):
with self.state:
self.paused = True # Block self.
def resume(self):
with self.state:
self.paused = False
self.state.notify() # Unblock self if waiting.
class Stopwatch(object):
""" Simple class to measure elapsed times. """
def start(self):
""" Establish reference point for elapsed time measurements. """
self.start_time = time.time()
return self
#property
def elapsed_time(self):
""" Seconds since started. """
try:
return time.time() - self.start_time
except AttributeError: # Wasn't explicitly started.
self.start_time = time.time()
return 0
MAX_RUN_TIME = 5 # Seconds.
concur = Concur()
stopwatch = Stopwatch()
print('Running for {} seconds...'.format(MAX_RUN_TIME))
concur.start()
while stopwatch.elapsed_time < MAX_RUN_TIME:
concur.resume()
# Can also do other concurrent operations here...
concur.pause()
# Do some other stuff...
# Show Concur thread executed.
print('concur.iterations: {}'.format(concur.iterations))
This is David Heffernan's idea fleshed-out. The example below runs for 1 second, then stops for 1 second, then runs for 1 second, and so on.
import time
import threading
import datetime as DT
import logging
logger = logging.getLogger(__name__)
def worker(cond):
i = 0
while True:
with cond:
cond.wait()
logger.info(i)
time.sleep(0.01)
i += 1
logging.basicConfig(level=logging.DEBUG,
format='[%(asctime)s %(threadName)s] %(message)s',
datefmt='%H:%M:%S')
cond = threading.Condition()
t = threading.Thread(target=worker, args=(cond, ))
t.daemon = True
t.start()
start = DT.datetime.now()
while True:
now = DT.datetime.now()
if (now-start).total_seconds() > 60: break
if now.second % 2:
with cond:
cond.notify()
The implementation of stop() would look like this:
def stop(self):
self.stopped = True
If you want to restart, then you can just create a new instance and start that.
while conditon:
inst = Concur()
inst.start()
#after some operation
inst.stop()
#some other operation
The documentation for Thread makes it clear that the start() method can only be called once for each instance of the class.
If you want to pause and resume a thread, then you'll need to use a condition variable.
Hey I'm learning psutil package and I want to know how to display current CPU usage when function is in progress? I suppose I need some threading or something like this, but how to do it? Thank u for any answers.
import psutil
import random
def iHateThis():
tab = []
for i in range(100000):
tab.append(random.randint(1, 10000))
tab.sort()
return tab;
while(True):
currentProcess = psutil.Process()
print(currentProcess.cpu_percent(interval=1))
You can use threading to run iHateThis or to run function with cpu_percent(). I choose second version. I will run cpu_percent() in thread.
Because it uses while True so thread would run forever and there wouldn't be nice method to stop thread so I use global variaable running with while running to have method to stop this loop.
import threading
import psutil
def display_cpu():
global running
running = True
currentProcess = psutil.Process()
# start loop
while running:
print(currentProcess.cpu_percent(interval=1))
def start():
global t
# create thread and start it
t = threading.Thread(target=display_cpu)
t.start()
def stop():
global running
global t
# use `running` to stop loop in thread so thread will end
running = False
# wait for thread's end
t.join()
and now I can use it to start and stop thread which will display CPU. Because I may have to stop process using Ctrl+C so it will raise error so I use try/finally to stop thread even if there will be error.
def i_hate_this():
tab = []
for i in range(1000000):
tab.append(random.randint(1, 10000))
tab.sort()
return tab
# ---
start()
try:
result = i_hate_this()
finally: # stop thread even if I press Ctrl+C
stop()
Full code:
import random
import threading
import psutil
def display_cpu():
global running
running = True
currentProcess = psutil.Process()
# start loop
while running:
print(currentProcess.cpu_percent(interval=1))
def start():
global t
# create thread and start it
t = threading.Thread(target=display_cpu)
t.start()
def stop():
global running
global t
# use `running` to stop loop in thread so thread will end
running = False
# wait for thread's end
t.join()
# ---
def i_hate_this():
tab = []
for i in range(1000000):
tab.append(random.randint(1, 10000))
tab.sort()
return tab
# ---
start()
try:
result = i_hate_this()
finally: # stop thread even if I press Ctrl+C
stop()
BTW: this can be converted to class which inherits from class Thread and then it can hide variable running in class.
import psutil
import random
import threading
class DisplayCPU(threading.Thread):
def run(self):
self.running = True
currentProcess = psutil.Process()
while self.running:
print(currentProcess.cpu_percent(interval=1))
def stop(self):
self.running = False
# ----
def i_hate_this():
tab = []
for i in range(1000000):
tab.append(random.randint(1, 10000))
tab.sort()
return tab
# ---
display_cpu = DisplayCPU()
display_cpu.start()
try:
result = i_hate_this()
finally: # stop thread even when I press Ctrl+C
display_cpu.stop()
It could be also converted to context manager to run it as
with display_cpu():
i_hate_this()
but I skip this part.
You can do this with the multiprocessing library. multiprocessing.Process is a class that represents a threaded process, is initiated with a function and name, and can be run at any time with .start().
import multiprocessing
import psutil
import random
def iHateThis():
tab = []
for i in range(100000):
tab.append(random.randint(1, 10000))
tab.sort()
return tab;
hate = multiprocessing.Process(name='hate', target=iHateThis)
hate.start()
while(True):
currentProcess = psutil.Process()
print(currentProcess.cpu_percent(interval=1))
I don't think you need to use psutil Process class as I think it is intended to be used to monitor a specific process. Using the code snippet from #furas (the accepted answer), you can do it with a thread like this:
def run(self):
self.run = True
while self.run:
psutil.cpu_percent(interval=1)
it works the same as the accepted answer in the following case:
_monitor.start()
try:
for i in range(50):
time.sleep(0.2)
finally:
_monitor.stop()
If you don't want to code it, I am doing it in a public repo if it can be of any help for someone: https://github.com/GTimothee/monitor
Let approach the problem differently and propose a decorator that can serve to measure CPU utilization while running
from functools import partial, wraps
def log_cpu_usage(func=None, msg_prefix: str = None):
"""
This function is a decorator that measures the execution time of a function and logs it.
"""
debug = True
if not debug:
return func
if func is None:
return partial(log_cpu_usage, msg_prefix=msg_prefix)
def new_func(data: mp.Queue, *args, **kwargs):
result = func(*args, **kwargs)
data.put(result)
#wraps(func)
def trace_execution(*args, **kwargs):
manager = mp.Queue() # to save return val between multi process
worker_process = mp.Process(target=new_func, args=(manager, *args), kwargs=kwargs)
worker_process.start()
p = psutil.Process(worker_process.pid)
cpu_percents = []
while worker_process.is_alive(): # while the subprocess is running
cpu_percents.append(p.cpu_percent() / psutil.cpu_count())
time.sleep(0.01)
worker_process.join()
ret_values = manager.get()
return sum(cpu_percents) / len(cpu_percents), ret_values
#log_cpu_usage
def iHateThis():
pass
How can I start and stop a thread with my poor thread class?
It is in loop, and I want to restart it again at the beginning of the code. How can I do start-stop-restart-stop-restart?
My class:
import threading
class Concur(threading.Thread):
def __init__(self):
self.stopped = False
threading.Thread.__init__(self)
def run(self):
i = 0
while not self.stopped:
time.sleep(1)
i = i + 1
In the main code, I want:
inst = Concur()
while conditon:
inst.start()
# After some operation
inst.stop()
# Some other operation
You can't actually stop and then restart a thread since you can't call its start() method again after its run() method has terminated. However you can make one pause and then later resume its execution by using a threading.Condition variable to avoid concurrency problems when checking or changing its running state.
threading.Condition objects have an associated threading.Lock object and methods to wait for it to be released and will notify any waiting threads when that occurs. Here's an example derived from the code in your question which shows this being done. In the example code I've made the Condition variable a part of Thread subclass instances to better encapsulate the implementation and avoid needing to introduce additional global variables:
from __future__ import print_function
import threading
import time
class Concur(threading.Thread):
def __init__(self):
super(Concur, self).__init__()
self.iterations = 0
self.daemon = True # Allow main to exit even if still running.
self.paused = True # Start out paused.
self.state = threading.Condition()
def run(self):
self.resume()
while True:
with self.state:
if self.paused:
self.state.wait() # Block execution until notified.
# Do stuff...
time.sleep(.1)
self.iterations += 1
def pause(self):
with self.state:
self.paused = True # Block self.
def resume(self):
with self.state:
self.paused = False
self.state.notify() # Unblock self if waiting.
class Stopwatch(object):
""" Simple class to measure elapsed times. """
def start(self):
""" Establish reference point for elapsed time measurements. """
self.start_time = time.time()
return self
#property
def elapsed_time(self):
""" Seconds since started. """
try:
return time.time() - self.start_time
except AttributeError: # Wasn't explicitly started.
self.start_time = time.time()
return 0
MAX_RUN_TIME = 5 # Seconds.
concur = Concur()
stopwatch = Stopwatch()
print('Running for {} seconds...'.format(MAX_RUN_TIME))
concur.start()
while stopwatch.elapsed_time < MAX_RUN_TIME:
concur.resume()
# Can also do other concurrent operations here...
concur.pause()
# Do some other stuff...
# Show Concur thread executed.
print('concur.iterations: {}'.format(concur.iterations))
This is David Heffernan's idea fleshed-out. The example below runs for 1 second, then stops for 1 second, then runs for 1 second, and so on.
import time
import threading
import datetime as DT
import logging
logger = logging.getLogger(__name__)
def worker(cond):
i = 0
while True:
with cond:
cond.wait()
logger.info(i)
time.sleep(0.01)
i += 1
logging.basicConfig(level=logging.DEBUG,
format='[%(asctime)s %(threadName)s] %(message)s',
datefmt='%H:%M:%S')
cond = threading.Condition()
t = threading.Thread(target=worker, args=(cond, ))
t.daemon = True
t.start()
start = DT.datetime.now()
while True:
now = DT.datetime.now()
if (now-start).total_seconds() > 60: break
if now.second % 2:
with cond:
cond.notify()
The implementation of stop() would look like this:
def stop(self):
self.stopped = True
If you want to restart, then you can just create a new instance and start that.
while conditon:
inst = Concur()
inst.start()
#after some operation
inst.stop()
#some other operation
The documentation for Thread makes it clear that the start() method can only be called once for each instance of the class.
If you want to pause and resume a thread, then you'll need to use a condition variable.
In a basic Unix-shell app, how would you print to stdout without disturbing any pending user input.
e.g. Below is a simple Python app that echos user input. A thread running in the background prints a counter every 1 second.
import threading, time
class MyThread( threading.Thread ):
running = False
def run(self):
self.running = True
i = 0
while self.running:
i += 1
time.sleep(1)
print i
t = MyThread()
t.daemon = True
t.start()
try:
while 1:
inp = raw_input('command> ')
print inp
finally:
t.running = False
Note how the thread mangles the displayed user input as they type it (e.g. hell1o wo2rld3). How would you work around that, so that the shell writes a new line while preserving the line the user's currently typing on?
You have to port your code to some way of controlling the terminal as slightly better than a teletype -- e.g. with the curses module in Python's standard library, or other ways to move the cursor away before emitting output, then move it back to where the user's busy inputting stuff.
You could defer writing output until just after you receive some input. For anything more advanced you'll have to use Alex's answer
import threading, time
output=[]
class MyThread( threading.Thread ):
running = False
def run(self):
self.running = True
i = 0
while self.running:
i += 1
time.sleep(1)
output.append(str(i))
t = MyThread()
t.daemon = True
t.start()
try:
while 1:
inp = raw_input('command> ')
while output:
print output.pop(0)
finally:
t.running = False