I wrote a Threading class which tests whether a webserver is up or not.
import urllib
import threading
import time
import Queue
class Thread_CheckDeviceState(threading.Thread):
def __init__(self, device_ip, queue, inter=0.1):
self._run = True
self._codes = {}
self._queue = queue
self._device_ip = device_ip
self._inter = inter
self._elapsed = 0
threading.Thread.__init__(self)
def stop(self):
self._run = False
def run(self):
start = time.time()
while self._run:
try:
code = urllib.urlopen(self._device_ip).getcode()
except Exception:
code = "nope"
finally:
measure = time.time()
self._elapsed += measure-start
print self._elapsed, code
self._codes.update(
{self._elapsed:code}
)
time.sleep(self._inter)
self._queue.put(self._codes)
q = Queue.Queue()
thread = Thread_CheckDeviceState("http://192.168.1.3", q)
thread.start()
time.sleep(10)
thread.stop()
print q.get()
It works fine - until I disconnect my pc from the network. From that moment on the thread just does nothing until it is stopped. I would expect it to just continue and set the code to "nope", like I wrote it in the exception handler. Why doesn't it work
You need to use urllib2 instead, and specify a timeout parameter when you call urlopen().
Related
I need to open multiple chrome drivers with selenium, then execute my script by threading in them.
How to make it wait until first threading is finished and then start second threading.
time.sleep(x) wont work for me, as I do not know how much time would first threading take and I need second threading to start as soon as first one is finished.
import time
import threading
from selenium import webdriver
mydrivers=[]
tabs = []
class ActivePool(object):
def __init__(self):
super(ActivePool, self).__init__()
self.active = []
self.lock = threading.Lock()
def makeActive(self, name):
with self.lock:
self.active.append(name)
def makeInactive(self, name):
with self.lock:
self.active.remove(name)
def main_worker(s):
#Driver State
global tabs
global mydrivers
mydrivers.append(webdriver.Chrome())
tabs.append(False)
def worker(s, pool):
with s:
global tabs
global mydrivers
name = threading.currentThread().getName()
pool.makeActive(name)
x = tabs.index(False)
tabs[x] = True
mydrivers[x].get("https://stackoverflow.com")
time.sleep(15)
pool.makeInactive(name)
tabs[x]= False
for k in range(5):
t = threading.Thread(target=main_worker, args=(k,))
t.start()
# How to make it wait until above threading is finished and then start below threading
pool = ActivePool()
s = threading.Semaphore(5)
for j in range(100):
t = threading.Thread(target=worker, name=j, args=(s, pool))
t.start()
thds = []
for k in range(5):
thds.append( threading.Thread(target=main_worker, args=(k,)))
for t in thds:
t.start()
for t in thds:
t.join()
Or, even:
thds = [threading.Thread(target=main_worker, args=(k,)) for k in range(5)]
for t in thds:
t.start()
for t in thds:
t.join()
To wait for a thread to finish you should use the thread.join function. Eg...
from threading import Thread
import time
def wait_sec():
time.sleep(2)
my_thread = Thread(target=wait_sec)
my_thread.start()
# after starting the thread join it to wait for end of target
my_thread.join()
print("You have waited 2 seconds")
Hey I'm learning psutil package and I want to know how to display current CPU usage when function is in progress? I suppose I need some threading or something like this, but how to do it? Thank u for any answers.
import psutil
import random
def iHateThis():
tab = []
for i in range(100000):
tab.append(random.randint(1, 10000))
tab.sort()
return tab;
while(True):
currentProcess = psutil.Process()
print(currentProcess.cpu_percent(interval=1))
You can use threading to run iHateThis or to run function with cpu_percent(). I choose second version. I will run cpu_percent() in thread.
Because it uses while True so thread would run forever and there wouldn't be nice method to stop thread so I use global variaable running with while running to have method to stop this loop.
import threading
import psutil
def display_cpu():
global running
running = True
currentProcess = psutil.Process()
# start loop
while running:
print(currentProcess.cpu_percent(interval=1))
def start():
global t
# create thread and start it
t = threading.Thread(target=display_cpu)
t.start()
def stop():
global running
global t
# use `running` to stop loop in thread so thread will end
running = False
# wait for thread's end
t.join()
and now I can use it to start and stop thread which will display CPU. Because I may have to stop process using Ctrl+C so it will raise error so I use try/finally to stop thread even if there will be error.
def i_hate_this():
tab = []
for i in range(1000000):
tab.append(random.randint(1, 10000))
tab.sort()
return tab
# ---
start()
try:
result = i_hate_this()
finally: # stop thread even if I press Ctrl+C
stop()
Full code:
import random
import threading
import psutil
def display_cpu():
global running
running = True
currentProcess = psutil.Process()
# start loop
while running:
print(currentProcess.cpu_percent(interval=1))
def start():
global t
# create thread and start it
t = threading.Thread(target=display_cpu)
t.start()
def stop():
global running
global t
# use `running` to stop loop in thread so thread will end
running = False
# wait for thread's end
t.join()
# ---
def i_hate_this():
tab = []
for i in range(1000000):
tab.append(random.randint(1, 10000))
tab.sort()
return tab
# ---
start()
try:
result = i_hate_this()
finally: # stop thread even if I press Ctrl+C
stop()
BTW: this can be converted to class which inherits from class Thread and then it can hide variable running in class.
import psutil
import random
import threading
class DisplayCPU(threading.Thread):
def run(self):
self.running = True
currentProcess = psutil.Process()
while self.running:
print(currentProcess.cpu_percent(interval=1))
def stop(self):
self.running = False
# ----
def i_hate_this():
tab = []
for i in range(1000000):
tab.append(random.randint(1, 10000))
tab.sort()
return tab
# ---
display_cpu = DisplayCPU()
display_cpu.start()
try:
result = i_hate_this()
finally: # stop thread even when I press Ctrl+C
display_cpu.stop()
It could be also converted to context manager to run it as
with display_cpu():
i_hate_this()
but I skip this part.
You can do this with the multiprocessing library. multiprocessing.Process is a class that represents a threaded process, is initiated with a function and name, and can be run at any time with .start().
import multiprocessing
import psutil
import random
def iHateThis():
tab = []
for i in range(100000):
tab.append(random.randint(1, 10000))
tab.sort()
return tab;
hate = multiprocessing.Process(name='hate', target=iHateThis)
hate.start()
while(True):
currentProcess = psutil.Process()
print(currentProcess.cpu_percent(interval=1))
I don't think you need to use psutil Process class as I think it is intended to be used to monitor a specific process. Using the code snippet from #furas (the accepted answer), you can do it with a thread like this:
def run(self):
self.run = True
while self.run:
psutil.cpu_percent(interval=1)
it works the same as the accepted answer in the following case:
_monitor.start()
try:
for i in range(50):
time.sleep(0.2)
finally:
_monitor.stop()
If you don't want to code it, I am doing it in a public repo if it can be of any help for someone: https://github.com/GTimothee/monitor
Let approach the problem differently and propose a decorator that can serve to measure CPU utilization while running
from functools import partial, wraps
def log_cpu_usage(func=None, msg_prefix: str = None):
"""
This function is a decorator that measures the execution time of a function and logs it.
"""
debug = True
if not debug:
return func
if func is None:
return partial(log_cpu_usage, msg_prefix=msg_prefix)
def new_func(data: mp.Queue, *args, **kwargs):
result = func(*args, **kwargs)
data.put(result)
#wraps(func)
def trace_execution(*args, **kwargs):
manager = mp.Queue() # to save return val between multi process
worker_process = mp.Process(target=new_func, args=(manager, *args), kwargs=kwargs)
worker_process.start()
p = psutil.Process(worker_process.pid)
cpu_percents = []
while worker_process.is_alive(): # while the subprocess is running
cpu_percents.append(p.cpu_percent() / psutil.cpu_count())
time.sleep(0.01)
worker_process.join()
ret_values = manager.get()
return sum(cpu_percents) / len(cpu_percents), ret_values
#log_cpu_usage
def iHateThis():
pass
Hey I'm learning psutil package and I want to know how to display current CPU usage when function is in progress? I suppose I need some threading or something like this, but how to do it? Thank u for any answers.
import psutil
import random
def iHateThis():
tab = []
for i in range(100000):
tab.append(random.randint(1, 10000))
tab.sort()
return tab;
while(True):
currentProcess = psutil.Process()
print(currentProcess.cpu_percent(interval=1))
You can use threading to run iHateThis or to run function with cpu_percent(). I choose second version. I will run cpu_percent() in thread.
Because it uses while True so thread would run forever and there wouldn't be nice method to stop thread so I use global variaable running with while running to have method to stop this loop.
import threading
import psutil
def display_cpu():
global running
running = True
currentProcess = psutil.Process()
# start loop
while running:
print(currentProcess.cpu_percent(interval=1))
def start():
global t
# create thread and start it
t = threading.Thread(target=display_cpu)
t.start()
def stop():
global running
global t
# use `running` to stop loop in thread so thread will end
running = False
# wait for thread's end
t.join()
and now I can use it to start and stop thread which will display CPU. Because I may have to stop process using Ctrl+C so it will raise error so I use try/finally to stop thread even if there will be error.
def i_hate_this():
tab = []
for i in range(1000000):
tab.append(random.randint(1, 10000))
tab.sort()
return tab
# ---
start()
try:
result = i_hate_this()
finally: # stop thread even if I press Ctrl+C
stop()
Full code:
import random
import threading
import psutil
def display_cpu():
global running
running = True
currentProcess = psutil.Process()
# start loop
while running:
print(currentProcess.cpu_percent(interval=1))
def start():
global t
# create thread and start it
t = threading.Thread(target=display_cpu)
t.start()
def stop():
global running
global t
# use `running` to stop loop in thread so thread will end
running = False
# wait for thread's end
t.join()
# ---
def i_hate_this():
tab = []
for i in range(1000000):
tab.append(random.randint(1, 10000))
tab.sort()
return tab
# ---
start()
try:
result = i_hate_this()
finally: # stop thread even if I press Ctrl+C
stop()
BTW: this can be converted to class which inherits from class Thread and then it can hide variable running in class.
import psutil
import random
import threading
class DisplayCPU(threading.Thread):
def run(self):
self.running = True
currentProcess = psutil.Process()
while self.running:
print(currentProcess.cpu_percent(interval=1))
def stop(self):
self.running = False
# ----
def i_hate_this():
tab = []
for i in range(1000000):
tab.append(random.randint(1, 10000))
tab.sort()
return tab
# ---
display_cpu = DisplayCPU()
display_cpu.start()
try:
result = i_hate_this()
finally: # stop thread even when I press Ctrl+C
display_cpu.stop()
It could be also converted to context manager to run it as
with display_cpu():
i_hate_this()
but I skip this part.
You can do this with the multiprocessing library. multiprocessing.Process is a class that represents a threaded process, is initiated with a function and name, and can be run at any time with .start().
import multiprocessing
import psutil
import random
def iHateThis():
tab = []
for i in range(100000):
tab.append(random.randint(1, 10000))
tab.sort()
return tab;
hate = multiprocessing.Process(name='hate', target=iHateThis)
hate.start()
while(True):
currentProcess = psutil.Process()
print(currentProcess.cpu_percent(interval=1))
I don't think you need to use psutil Process class as I think it is intended to be used to monitor a specific process. Using the code snippet from #furas (the accepted answer), you can do it with a thread like this:
def run(self):
self.run = True
while self.run:
psutil.cpu_percent(interval=1)
it works the same as the accepted answer in the following case:
_monitor.start()
try:
for i in range(50):
time.sleep(0.2)
finally:
_monitor.stop()
If you don't want to code it, I am doing it in a public repo if it can be of any help for someone: https://github.com/GTimothee/monitor
Let approach the problem differently and propose a decorator that can serve to measure CPU utilization while running
from functools import partial, wraps
def log_cpu_usage(func=None, msg_prefix: str = None):
"""
This function is a decorator that measures the execution time of a function and logs it.
"""
debug = True
if not debug:
return func
if func is None:
return partial(log_cpu_usage, msg_prefix=msg_prefix)
def new_func(data: mp.Queue, *args, **kwargs):
result = func(*args, **kwargs)
data.put(result)
#wraps(func)
def trace_execution(*args, **kwargs):
manager = mp.Queue() # to save return val between multi process
worker_process = mp.Process(target=new_func, args=(manager, *args), kwargs=kwargs)
worker_process.start()
p = psutil.Process(worker_process.pid)
cpu_percents = []
while worker_process.is_alive(): # while the subprocess is running
cpu_percents.append(p.cpu_percent() / psutil.cpu_count())
time.sleep(0.01)
worker_process.join()
ret_values = manager.get()
return sum(cpu_percents) / len(cpu_percents), ret_values
#log_cpu_usage
def iHateThis():
pass
I have simple script for watchdog on network device. Script monitors response from PING command. If there is no answer then second thread executes and first thread is stopped. If second thread is finished then first thread is resumed (checking ping). If there is no answer then following message appears:
RuntimeError: threads can only be started once
Here is my code:
#!/usr/bin/python
import os
import time
import sqlite3
from ablib import Pin
import threading
led=Pin('W9','OUTPUT')
class threadout1(threading.Thread):
def run(self):
while True:
conn = sqlite3.connect('database/database.db')
cur = conn.cursor()
cur.execute("SELECT * FROM watchdog")
rows_output = cur.fetchall()
time.sleep(1)
if rows_output[0][1] == "ping":
response = os.system("ping -c 1 " + rows_output[0][2])
if response != 0:
print "bad"
rest.start()
rest.join()
class restart(threading.Thread):
def run(self):
led.on()
time.sleep(15)
led.off()
thr = threadout1()
rest = restart()
thr.start()
You can either create the restart thread every time you need it
if response != 0:
print "bad"
restart_thread = restart()
restart_thread.start()
restart_thread.join()
or use Events
class restart_thread(threading.Thread):
def __init__(self, evt):
self.evt = evt
def run(self):
self.evt.wait()
# do stuff
self.evt.clear()
class threadout(threading.Thread):
def __init__(self, evt):
self.evt = evt
def run(self):
if #other thread needs to run once
self.evt.set()
evt = threading.Event()
restart_thread = restart(evt)
restart_thread.start()
pinging_thread = threadout(evt)
pinging_thread.start()
To make the pinging_thread wait for the restart_thread to finish, you could use another Event.
Let say I have a code like this:
def func1(a,b,c):
try:
p = pycurl.Curl()
p.setopt(pycurl.PROXY, "127.0.0.1")
p.setopt(pycurl.PROXYPORT, 9050)
p.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5)
p.perform()
p.close()
except pycurl.error as error:
if error[0] == 28: # timeout - change proxy
print "Tor timeout, need to change"
queue.put((a,b,c))
new_tor()
return
def new_tor():
# send_signal_for_new_ident_is_here
I start this code in 7 threads.
And when a thread receives error 28 it change the identify.
But it happens that ALL 7 THREADS sending signal to change identify.
How to do this:
If thread received error 28, then it calls new_tor() and other 6 threads don't but waiting for result and only then they proceed to work. How to sync this?
Just put an error "id" into the queue and if you encounter it, put the value back into the queue, and then handle as needed.
You don't wish to end the thread, which is what I did.
So, you can have some unique identifier for each thread, such that if once a thread encounters an error, it also adds the data (it's identifier) that says it encountered this error before, so that if all threads have encountered this error, the error is removed from the queue.
Code:
import threading
import Queue
y = 0
def f1():
global y
y += 1
if y > 100:
raise ValueError('trial')
def f2():
return
class Test(threading.Thread):
def __init__(self, func, name):
threading.Thread.__init__(self)
self.func = func
self.name = name
def run(self):
while True:
x = ''
if not queue.empty():
x = queue.get()
if x == 'error':
queue.put(x)
print 'Stopping %s' % (self.name,)
return
try:
self.func()
except Exception as e:
queue.put('error')
queue = Queue.Queue()
thread1 = Test(f1, '1')
thread2 = Test(f2, '2')
thread1.start()
thread2.start()