Given the code below
from threading import Thread
import Queue
from time import sleep
class myClassA(Thread):
def __init__(self,num,q):
Thread.__init__(self)
self.daemon = True
self.num = num
self.start()
def run(self):
while True:
self.num = self.num+1
q.put(self.num)
sleep(5)
class myClassB(Thread):
def __init__(self,num,q):
Thread.__init__(self)
self.daemon = True
self.num = num
self.start()
def run(self):
while True:
self.num = q.get()
print self.num
sleep(1)
num = 0
q = Queue.Queue()
myClassA(num,q)
myClassB(num,q)
while True:
pass
Why doesn't Class B print every second? I would expect Class B to print five 1's then five 2's etc. Is q.get() a blocking function?
Yes, Queue.get() is blocking by default. From the documentation:
If optional args block is true and timeout is None (the default), block if necessary until an item is available.
Bold emphasis mine. Because q.get() blocks, it won't return until the other thread has put something in the queue for it to fetch.
Even so, removing an item from the queue means it won't be there the next time. q.get() doesn't leave the number there to be fetched again and again.
Instead, if you were to use q.get(False) (or used q.get_nowait()) to prevent blocking, an Empty exception is raised instead.
Related
my tool stops randomly and it seems like all threads are 'ghosts'.
How does it work:
The tool loops until the max number of allowed threads at the same time are running, in this case 20. When a thread finishes it starts the next one.
Problem:
After like an hour of doing this, the tool is stuck at 20 Threads running but nothing happens anymore.
Thanks in advance everyone!
maxthreadcount = 20
while True:
if threading.active_count() < maxthreadcount:
threading.Thread(target=Dealer).start()
Dealer:
def Dealer():
print("thread started")
return
You need to terminate previously created threads after their job (print command in this case) is done.
Take a look at this example from this article:
class CountdownTask:
def __init__(self):
self._running = True
def terminate(self):
self._running = False
def run(self, n):
while self._running and n > 0:
print('T-minus', n)
n -= 1
time.sleep(5)
c = CountdownTask()
t = Thread(target = c.run, args =(10, ))
t.start()
...
# Signal termination
c.terminate()
# Wait for actual termination (if needed)
t.join()
I think you should call self.terminate() after doing the print. Something like below:
class Dealer():
def __init__(self):
self._running = True
def run(self):
print("thread started")
return self.terminate()
def terminate(self):
self._running = False
Edit
I also believe you can make use of python's ThreadPool to this extent. Instead of spawning threads yourself, you might be able to reuse threads after their assigned task is over, for the new tasks.
I have a little doubt if one could solve my issue, and create successful communication between threads.
First example and this is how it should be working, but does not work well:
import Queue,threading,time
class th(threading.Thread):
def __init__(self,q):
threading.Thread.__init__(self)
self.q = q
self.t = time
def run(self):
for i in range(5):
self.q.put(i)
self.t.sleep(0.5) # <----------
self.q.put('end')
class main(object):
def __init__(self):
self.q = Queue.Queue()
self.thread = th(self.q)
self.thread.setDaemon(True)
self.thread.run()
self.call()
def call(self):
while True:
recv = self.q.get();
if recv == 'end':
break
else:
print recv
if __name__ == '__main__':
root = main()
root.call()
In this example, all printed at the same time:
0,1,2,3,4
Second example:
import Queue,threading,time
class th(threading.Thread):
def __init__(self,q):
threading.Thread.__init__(self);
self.q = q
self.t = time
def run(self):
for i in range(5):
self.q.put(i) # <------ no sleep()
self.q.put('end')
class main(object):
def __init__(self):
self.q = Queue.Queue()
self.thread = th(self.q)
self.thread.setDaemon(True)
self.thread.run()
self.call()
def call(self):
while True:
recv = self.q.get()
if recv == 'end':
break
else:
print recv
if __name__ == '__main__':
root = main()
root.call()
the code is printed as it has to
0,
1
2
3
4
one to one
is there any way that the sleep function in the same way?
You don't want to call the run method on a thread directly. Call start instead, which will kick off the child thread, which will in turn run the run method.
Your current code is essentially single threaded, since the run call does the work of the child thread in the parent instead. The child thread is never actually started! (You're also calling your main.call method twice, which I'd expect to block or raise an exception, but that's a separate issue.)
sorry, it was something very simple, really simple, just had to replace
self.thread.run()
by
self.threat.start()
I've been looking into a way to directly change variables in a running module.
What I want to achieve is that a load test is being run and that I can manually adjust the call pace or whatsoever.
Below some code that I just created (not-tested e.d.), just to give you an idea.
class A():
def __init__(self):
self.value = 1
def runForever(self):
while(1):
print self.value
def setValue(self, value):
self.value = value
if __name__ == '__main__':
#Some code to create the A object and directly apply the value from an human's input
a = A()
#Some parallelism or something has to be applied.
a.runForever()
a.setValue(raw_input("New value: "))
Edit #1: Yes, I know that now I will never hit the a.setValue() :-)
Here is a multi-threaded example. This code will work with the python interpreter but not with the Python Shell of IDLE, because the raw_input function is not handled the same way.
from threading import Thread
from time import sleep
class A(Thread):
def __init__(self):
Thread.__init__(self)
self.value = 1
self.stop_flag = False
def run(self):
while not self.stop_flag:
sleep(1)
print(self.value)
def set_value(self, value):
self.value = value
def stop(self):
self.stop_flag = True
if __name__ == '__main__':
a = A()
a.start()
try:
while 1:
r = raw_input()
a.set_value(int(r))
except:
a.stop()
The pseudo code you wrote is quite similar to the way Threading / Multiprocessing works in python. You will want to start a (for example) thread that "runs forever" and then instead of modifying the internal rate value directly, you will probably just send a message through a Queue that gives the new value.
Check out this question.
Here is a demonstration of doing what you asked about. I prefer to use Queues to directly making calls on threads / processes.
import Queue # !!warning. if you use multiprocessing, use multiprocessing.Queue
import threading
import time
def main():
q = Queue.Queue()
tester = Tester(q)
tester.start()
while True:
user_input = raw_input("New period in seconds or (q)uit: ")
if user_input.lower() == 'q':
break
try:
new_speed = float(user_input)
except ValueError:
new_speed = None # ignore junk
if new_speed is not None:
q.put(new_speed)
q.put(Tester.STOP_TOKEN)
class Tester(threading.Thread):
STOP_TOKEN = '<<stop>>'
def __init__(self, q):
threading.Thread.__init__(self)
self.q = q
self.speed = 1
def run(self):
while True:
# get from the queue
try:
item = self.q.get(block=False) # don't hang
except Queue.Empty:
item = None # do nothing
if item:
# stop when requested
if item == self.STOP_TOKEN:
break # stop this thread loop
# otherwise check for a new speed
try:
self.speed = float(item)
except ValueError:
pass # whatever you like with unknown input
# do your thing
self.main_code()
def main_code(self):
time.sleep(self.speed) # or whatever you want to do
if __name__ == '__main__':
main()
Let say I have a code like this:
def func1(a,b,c):
try:
p = pycurl.Curl()
p.setopt(pycurl.PROXY, "127.0.0.1")
p.setopt(pycurl.PROXYPORT, 9050)
p.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5)
p.perform()
p.close()
except pycurl.error as error:
if error[0] == 28: # timeout - change proxy
print "Tor timeout, need to change"
queue.put((a,b,c))
new_tor()
return
def new_tor():
# send_signal_for_new_ident_is_here
I start this code in 7 threads.
And when a thread receives error 28 it change the identify.
But it happens that ALL 7 THREADS sending signal to change identify.
How to do this:
If thread received error 28, then it calls new_tor() and other 6 threads don't but waiting for result and only then they proceed to work. How to sync this?
Just put an error "id" into the queue and if you encounter it, put the value back into the queue, and then handle as needed.
You don't wish to end the thread, which is what I did.
So, you can have some unique identifier for each thread, such that if once a thread encounters an error, it also adds the data (it's identifier) that says it encountered this error before, so that if all threads have encountered this error, the error is removed from the queue.
Code:
import threading
import Queue
y = 0
def f1():
global y
y += 1
if y > 100:
raise ValueError('trial')
def f2():
return
class Test(threading.Thread):
def __init__(self, func, name):
threading.Thread.__init__(self)
self.func = func
self.name = name
def run(self):
while True:
x = ''
if not queue.empty():
x = queue.get()
if x == 'error':
queue.put(x)
print 'Stopping %s' % (self.name,)
return
try:
self.func()
except Exception as e:
queue.put('error')
queue = Queue.Queue()
thread1 = Test(f1, '1')
thread2 = Test(f2, '2')
thread1.start()
thread2.start()
How can I start and stop a thread with my poor thread class?
It is in loop, and I want to restart it again at the beginning of the code. How can I do start-stop-restart-stop-restart?
My class:
import threading
class Concur(threading.Thread):
def __init__(self):
self.stopped = False
threading.Thread.__init__(self)
def run(self):
i = 0
while not self.stopped:
time.sleep(1)
i = i + 1
In the main code, I want:
inst = Concur()
while conditon:
inst.start()
# After some operation
inst.stop()
# Some other operation
You can't actually stop and then restart a thread since you can't call its start() method again after its run() method has terminated. However you can make one pause and then later resume its execution by using a threading.Condition variable to avoid concurrency problems when checking or changing its running state.
threading.Condition objects have an associated threading.Lock object and methods to wait for it to be released and will notify any waiting threads when that occurs. Here's an example derived from the code in your question which shows this being done. In the example code I've made the Condition variable a part of Thread subclass instances to better encapsulate the implementation and avoid needing to introduce additional global variables:
from __future__ import print_function
import threading
import time
class Concur(threading.Thread):
def __init__(self):
super(Concur, self).__init__()
self.iterations = 0
self.daemon = True # Allow main to exit even if still running.
self.paused = True # Start out paused.
self.state = threading.Condition()
def run(self):
self.resume()
while True:
with self.state:
if self.paused:
self.state.wait() # Block execution until notified.
# Do stuff...
time.sleep(.1)
self.iterations += 1
def pause(self):
with self.state:
self.paused = True # Block self.
def resume(self):
with self.state:
self.paused = False
self.state.notify() # Unblock self if waiting.
class Stopwatch(object):
""" Simple class to measure elapsed times. """
def start(self):
""" Establish reference point for elapsed time measurements. """
self.start_time = time.time()
return self
#property
def elapsed_time(self):
""" Seconds since started. """
try:
return time.time() - self.start_time
except AttributeError: # Wasn't explicitly started.
self.start_time = time.time()
return 0
MAX_RUN_TIME = 5 # Seconds.
concur = Concur()
stopwatch = Stopwatch()
print('Running for {} seconds...'.format(MAX_RUN_TIME))
concur.start()
while stopwatch.elapsed_time < MAX_RUN_TIME:
concur.resume()
# Can also do other concurrent operations here...
concur.pause()
# Do some other stuff...
# Show Concur thread executed.
print('concur.iterations: {}'.format(concur.iterations))
This is David Heffernan's idea fleshed-out. The example below runs for 1 second, then stops for 1 second, then runs for 1 second, and so on.
import time
import threading
import datetime as DT
import logging
logger = logging.getLogger(__name__)
def worker(cond):
i = 0
while True:
with cond:
cond.wait()
logger.info(i)
time.sleep(0.01)
i += 1
logging.basicConfig(level=logging.DEBUG,
format='[%(asctime)s %(threadName)s] %(message)s',
datefmt='%H:%M:%S')
cond = threading.Condition()
t = threading.Thread(target=worker, args=(cond, ))
t.daemon = True
t.start()
start = DT.datetime.now()
while True:
now = DT.datetime.now()
if (now-start).total_seconds() > 60: break
if now.second % 2:
with cond:
cond.notify()
The implementation of stop() would look like this:
def stop(self):
self.stopped = True
If you want to restart, then you can just create a new instance and start that.
while conditon:
inst = Concur()
inst.start()
#after some operation
inst.stop()
#some other operation
The documentation for Thread makes it clear that the start() method can only be called once for each instance of the class.
If you want to pause and resume a thread, then you'll need to use a condition variable.