The following code executes two threads (multithread), each with different time delays so that each thread will finish at a different time.
Once both threads are finished module display1.py issues a print statement saying they are BOTH finished.
I would like module display1.py to issue a 'finished' statement for EACH thread AS EACH thread finishes
How can i do this ... amendments to my working code appreciated! I'd like to change as little of the current code as possible so a better form of variable transfer between the two modules might be what I'm after
display1.py
from threads1 import *
manager = ThreadManager()
manager.start(False)
print (manager.GetResults())
threads1.py
from threading import Thread
import time
class ThreadManager:
def __init__(self):
pass
def start(self, answer):
self.answer = answer
thread_refs = []
t1 = MyThread(70, 'Not finished')
t1.daemon = True
t1.start()
t2 = MyThread(2, 'Not finished')
t2.daemon = True
t2.start()
while True:
if t1.AskFinished == 'Finished' and t2.AskFinished == 'Finished': #If I break the loop after EACH site, Only the first to finish will be sent via GetResults to display1.py
global results
results = [t1.AskFinished, t2.AskFinished]
print("Both Finished")
break
def GetResults(self):
global results
return(results)
class MyThread(Thread):
def __init__(self, SleepWait, AskFinished):
Thread.__init__(self)
self.SleepWait = SleepWait
self.AskFinished = AskFinished
def run(self):
time.sleep(self.SleepWait)
self.AskFinished = 'Finished'
What you have here (entering a very tight check loop in the main thread) is a very naive approach to threading in many languages, but especially in python where GIL contention will just slow the threads down a great bit.
What is a better idea is instead using queue.Queue to push info when a thread is completed. This allows the main thread to block on the queue instead, which is less CPU intensive as well as allowing you to know (out of order) which one is finished.
The changes you would need to make:
at the top of the module threads1.py:
import queue
finished_queue = queue.Queue()
in your start():
num_finished = 0
while True:
info = finished_queue.get()
num_finished += 1
if info is t1:
print("t1 finished")
else:
print("t2 finished")
if num_finished == 2:
global results
results = [t1.AskFinished, t2.AskFinished]
print("Both Finished")
break
and finally in run():
def run(self):
time.sleep(self.SleepWait)
self.AskFinished = 'Finished'
finished_queue.put(self)
Some more fundamental modifications I'd make is actually pushing the result into the queue and then fetching the results out, skipping the extra step before GetResults. Furthermore, if GetResults had to stay, I'd pass them through a field on self e.g. self.results = [t1.AskFinished, t2.AskFinished]
Update:
Ok, so you want to know more about how to have display1.py print the results. It would be helpful if you could explain why it matters, because that might make a difference in how you should do this, but here's a first approach:
# threads1.py
from threading import Thread
import time
class ThreadManager:
def __init__(self):
self.threads = {}
def start(self):
t1 = MyThread(4)
t1.daemon = True
t1.start()
self.threads[1] = t1
t2 = MyThread(1)
t2.daemon = True
t2.start()
self.threads[2] = t2
def is_alive(self, thread_id):
return self.threads[thread_id].is_alive()
def GetResults(self): # or you could just access results directly
return self.results
class MyThread(Thread):
def __init__(self, SleepWait):
Thread.__init__(self)
self.SleepWait = SleepWait
def run(self):
time.sleep(self.SleepWait)
And then...
# display1.py
from threads1 import *
manager = ThreadManager()
manager.start()
t1_state = t2_state = True
while manager.is_alive(1) or manager.is_alive(2):
time.sleep(1)
if manager.is_alive(1) != t1_state:
print("t1 finished")
t1_state = manager.is_alive(1)
if manager.is_alive(2) != t2_state:
print("t2 finished")
t2_state = manager.is_alive(2)
if not manager.is_alive(1) and not manager.is_alive(2):
print("Both Finished")
break
You should eventually consider using a Queue as suggested by Crast; but let's focus on getting this right first.
Original Post:
There are a number of problems with this code.
First, you should use t1.is_alive() to check if a thread is finished. There's no need to reimplement it with AskFinished.
Second, the while True: loop in threads1.py is doing nothing at a furious rate while it waits for your threads to terminate. Take a look at the cpu usage while this is running if you don't believe me. You should throw a time.sleep(1) statement in there.
Third, why are you using a global var to return your results? That's a really strange thing to do. Just store it in self!
And finally, why does display1.py have to print the messages? Why can't thread1.py do that?
With these four points in mind, here's a thread1.py that works more sensibly:
from threading import Thread
import time
class ThreadManager:
def __init__(self):
self.results = None
def start(self, answer): # why is "answer" here?
self.answer = answer
thread_refs = []
t1 = MyThread(4, 'Not finished')
t1.daemon = True
t1.start()
t2 = MyThread(1, 'Not finished')
t2.daemon = True
t2.start()
t1_state = t2_state = True
while t1.is_alive() or t2.is_alive():
time.sleep(1)
if t1.is_alive() != t1_state:
print("t1 finished")
t1_state = t1.is_alive()
if t2.is_alive() != t2_state:
print("t2 finished")
t2_state = t2.is_alive()
if not t1.is_alive() and not t2.is_alive():
self.results = [t1.AskFinished, t2.AskFinished]
print("Both Finished")
break
def GetResults(self): # or you could just access results directly
return self.results
class MyThread(Thread):
def __init__(self, SleepWait, AskFinished):
Thread.__init__(self)
self.SleepWait = SleepWait
self.AskFinished = AskFinished
def run(self):
time.sleep(self.SleepWait)
self.AskFinished = 'Finished'
Now, this still doesn't do exactly what you wanted, because you asked for display.py to do the displaying. To make that work, you'd have to put your while True loop in display.py and add an ThreadManager.is_alive() method that display.py could use to check whether a thread is alive or not. If you want to see how to do that let me know.
Im not familiar with threading but since no answers yet ill give it a shot.
In this:
Cant you just add two if statements before hand?
while True:
if t1.askFinished == 'Finished':
print("t1 Finished")
if t2.askFinished == 'Finished':
print("t2 Finished")
if t1.AskFinished == 'Finished' and t2.AskFinished == 'Finished': #If I break the loop after EACH site, Only the first to finish will be sent via GetResults to display1.py
global results
results = [t1.AskFinished, t2.AskFinished]
print("Both Finished")
break
edit: I tried changing your code as little as possible... it's not very well written though tbh. (No offense)
Related
This simple code example:
import threading
import time
class Monitor():
def __init__(self):
self.stop = False
self.blocked_emails = []
def start_monitor(self):
print("Run start_monitor")
rows = []
while not self.stop:
self.check_rows(rows)
print("inside while")
time.sleep(1)
def check_rows(self, rows):
print('check_rows')
def stop_monitoring(self):
print("Run stop_monitoring")
self.stop = True
if __name__ == '__main__':
monitor = Monitor()
b = threading.Thread(name='background_monitor', target=monitor.start_monitor())
b.start()
b.join()
for i in range(0, 10):
time.sleep(2)
print('Wait 2 sec.')
monitor.stop_monitoring()
How can I run background thread, in mine case background_monitor without blocking main thread?
I wanted to background_monitor thread stopped on after stop_monitoring will be called
I mine example, the for loop from main thread never called and the background is running forever.
There are two issues with your current code. Firstly, you're calling monitor.start_monitor on this line, whereas according to the docs
target is the callable object to be invoked by the run() method. Defaults to None, meaning nothing is called
This means that you need to pass it as a function rather than calling it. To fix this, you should change the line
b = threading.Thread(name='background_monitor', target=monitor.start_monitor())
to
b = threading.Thread(name='background_monitor', target=monitor.start_monitor)
which passes the function as an argument.
Secondly, you use b.join() before stopping the thread, which waits for the second thread to finish before continuing. Instead, you should place that below the monitor.stop_monitoring().
The corrected code looks like this:
import threading
import time
class Monitor():
def __init__(self):
self.stop = False
self.blocked_emails = []
def start_monitor(self):
print("Run start_monitor")
rows = []
while not self.stop:
self.check_rows(rows)
print("inside while")
time.sleep(1)
def check_rows(self, rows):
print('check_rows')
def stop_monitoring(self):
print("Run stop_monitoring")
self.stop = True
if __name__ == '__main__':
monitor = Monitor()
b = threading.Thread(name='background_monitor', target=monitor.start_monitor)
b.start()
for i in range(0, 10):
time.sleep(2)
print('Wait 2 sec.')
monitor.stop_monitoring()
b.join()
EDIT 3: See last example at the end.
I need a while loop doing continuous send and return operations with an USB connection.
During this continuous operation I need (amongst other stuff in my main script) a few identical and isolated send/return operations on that same USB connection.
This seems to require multiprocessing and some tweaking.
I want to use the following workaround with the multiprocessing library:
Put the continuous send/return operation on a different thread with a pool (apply_async).
Put this process on "hold" when I perform the isolated send/return operation (using clear()).
Immediately after the isolated send/return operation resume the continuous send/return (using set()).
Stop the continuous send/return when i reach the end of the main script (here i have no solution yet should be x.stop() or something like this since terminate() won't do).
Get some return value from the stopped process (use get()).
I tried couple of things already but i just cant exit the while loop via a main command.
import multiprocessing
import time
def setup(event):
global unpaused
unpaused = event
class func:
def __init__(self):
self.finished = False
def stop(self):
self.finished = True
def myFunction(self, arg):
i = 0
s=[]
while self.finished == False:
unpaused.wait()
print(arg+i)
s.append(arg+i)
i=i+1
time.sleep(1)
return s
if __name__ == "__main__":
x=func()
event = multiprocessing.Event() # initially unset, so workers will be paused at first
pool = multiprocessing.Pool(1, setup, (event,))
result = pool.apply_async(x.myFunction, (10,))
print('We unpause for 2 sec')
event.set() # unpause
time.sleep(2)
print('We pause for 2 sec')
event.clear() # pause
time.sleep(2)
print('We unpause for 2 sec')
event.set() # unpause
time.sleep(2)
print('Now we try to terminate in 2 sec')
time.sleep(2)
x.stop()
return_val = result.get()
print('get worked with '+str(return_val))
Can someone point me in the right direction? As seen this wont stop with x.stop().
Global values also do not work.
Thanks in advance.
EDIT:
as suggested I tried to put the multiprocessing in a seperated object.
Is this done by putting functions in a class like my example below?
import multiprocessing
import time
class func(object):
def __init__(self):
self.event = multiprocessing.Event() # initially unset, so workers will be paused at first
self.pool = multiprocessing.Pool(1, self.setup, (self.event,))
def setup(self):
global unpaused
unpaused = self.event
def stop(self):
self.finished = True
def resume(self):
self.event.set() # unpause
def hold(self):
self.event.clear() #pause
def run(self, arg):
self.pool.apply_async(self.myFunction, (arg,))
def myFunction(self, arg):
i = 0
s=[]
self.finished = False
while self.finished == False:
unpaused.wait()
print(arg+i)
s.append(arg+i)
i=i+1
time.sleep(1)
return s
if __name__ == "__main__":
x=func()
result = x.run(10)
print('We unpause for 2 sec')
x.resume() # unpause
time.sleep(2)
print('We pause for 2 sec')
x.hold() # pause
time.sleep(2)
print('We unpause for 2 sec')
x.resume() # unpause
time.sleep(2)
print('Now we try to terminate in 2 sec')
time.sleep(2)
x.stop()
return_val = result.get()
print('get worked with '+str(return_val))
I added a hold and resume function and put the setup function in a single class.
But the lower example wont even run the function anymore.
What a complex little problem. I am puzzled with this.
EDIT2:
I tried a workaround with what i found so far.
Big trouble came in while using the microprocessing.pool library.
It is not straightforward using it with the USB connection...
I produced a mediocre workaround below:
from multiprocessing.pool import ThreadPool
import time
class switch:
state = 1
s1 = switch()
def myFunction(arg):
i = 0
while s1.state == 1 or s1.state == 2 or s1.state == 3:
if s1.state == 1:
print(arg+i)
s.append(arg+i)
i=i+1
time.sleep(1)
elif s1.state == 2:
print('we entered snippet mode (state 2)')
time.sleep(1)
x = s
return x
pool.close()
pool.join()
elif s1.state == 3:
while s1.state == 3:
time.sleep(1)
print('holding (state 3)')
return s
if __name__ == "__main__":
global s
s=[]
print('we set the state in the class on top to ' +str(s1.state))
pool = ThreadPool(processes=1)
async_result = pool.apply_async(myFunction, (10,))
print('in 5 sec we switch mode sir, buckle up')
time.sleep(5)
s1.state = 2
print('we switched for a snippet which is')
snippet = async_result.get()
print(str(snippet[-1])+' this snippet comes from main')
time.sleep(1)
print('now we return to see the full list in the end')
s1.state = 1
async_result = pool.apply_async(myFunction, (10,))
print('in 5 sec we hold it')
time.sleep(5)
s1.state = 3
print('in 5 sec we exit')
time.sleep(5)
s1.state = 0
return_val = async_result.get()
print('Succsses if you see a list of numbers '+ str(return_val))
EDIT 3:
from multiprocessing.pool import ThreadPool
import time
class switch:
state = 1
s1 = switch()
def myFunction(arg):
i = 0
while s1.state == 1 or s1.state == 2:
if s1.state == 1:
print(arg+i)
s.append(arg+i)
i=i+1
time.sleep(1)
elif s1.state == 2:
print('we entered snippet mode (state 2)')
time.sleep(1)
x = s
return x
pool.close() #These are not relevant i guess.
pool.join() #These are not relevant i guess.
return s
if __name__ == "__main__":
global s
s=[]
print('we set the state in the class on top to ' +str(s1.state))
pool = ThreadPool(processes=1)
async_result = pool.apply_async(myFunction, (10,))
print('in 5 sec we switch mode sir, buckle up')
time.sleep(5)
s1.state = 2
snippet = async_result.get()
print(str(snippet[-1])+' this snippet comes from main')
time.sleep(1)
print('now we return to see the full list in the end')
s1.state = 1
async_result = pool.apply_async(myFunction, (10,))
print('in 5 sec we exit')
time.sleep(5)
s1.state = 0
return_val = async_result.get()
print('Succsses if you see a list of numbers '+ str(return_val))
Well, this is what i have come up with...
Not great not terrible. Maybe a bit more on the terrible side (:
I hate it that I have to recall the function pool.apply_async(myFunction, (10,)) after I grabbed a single piece of data.
Currently only ThreadingPool works with no further code changes in my actual script!
in a situation where I need a process to run continuously, while occasionally doing other things, I like to use asyncio. This is a rough draft of how I would approach this
import asyncio
class MyObject:
def __init__(self):
self.mydatastructure = []
self.finished = False
self.loop = None
async def main_loop(self):
while not self.finished:
new_result = self.get_data()
self.mydatastructure.append(new_result)
await asyncio.sleep(0)
async def timed_loop(self):
while not self.finished:
await asyncio.sleep(2)
self.dotimedtask(self.mydatastructure)
async def run(self):
await asyncio.gather(self.main_loop(), self.timed_loop())
asyncio.run(MyObject().run())
only one coroutine will be running at a time, with the timed one being scheduled once every 2 seconds. It would always get the data passed out of the most recent continuous execution. you could do things like keep a connection open on an object as well. Depending on your requirements (is it a 2 second interval, or once every other second no matter how long it takes) there are library packages to make the scheduling a bit more elegant.
my tool stops randomly and it seems like all threads are 'ghosts'.
How does it work:
The tool loops until the max number of allowed threads at the same time are running, in this case 20. When a thread finishes it starts the next one.
Problem:
After like an hour of doing this, the tool is stuck at 20 Threads running but nothing happens anymore.
Thanks in advance everyone!
maxthreadcount = 20
while True:
if threading.active_count() < maxthreadcount:
threading.Thread(target=Dealer).start()
Dealer:
def Dealer():
print("thread started")
return
You need to terminate previously created threads after their job (print command in this case) is done.
Take a look at this example from this article:
class CountdownTask:
def __init__(self):
self._running = True
def terminate(self):
self._running = False
def run(self, n):
while self._running and n > 0:
print('T-minus', n)
n -= 1
time.sleep(5)
c = CountdownTask()
t = Thread(target = c.run, args =(10, ))
t.start()
...
# Signal termination
c.terminate()
# Wait for actual termination (if needed)
t.join()
I think you should call self.terminate() after doing the print. Something like below:
class Dealer():
def __init__(self):
self._running = True
def run(self):
print("thread started")
return self.terminate()
def terminate(self):
self._running = False
Edit
I also believe you can make use of python's ThreadPool to this extent. Instead of spawning threads yourself, you might be able to reuse threads after their assigned task is over, for the new tasks.
I'm new to threading and queue. So, I wrote this program where I'm trying to return a value to one of the 2 functions in the threading class MyThread and print/process the returned value in the main() function :
import threading
import datetime
import Queue
a=Queue.Queue()
b=Queue.Queue()
q=Queue.Queue()
class MyThread(threading.Thread):
def __init__(self,q):
threading.Thread.__init__(self)
self.que=q
t1=threading.Thread(target=self.prints,args=(4,))
t2=threading.Thread(target=self.printy,args=(6,self.que,))
t1.start()
t2.start()
item=self.que.get()
print(item)
print "*"*30
it=item*2
print(it)
t1.join()
t2.join()
def main(self):
t3=threading.Thread(target=self.prints,args=(3,))
t4=threading.Thread(target=self.printy,args=(5,self.que,))
t3.start()
t4.start()
item=self.que.get()
print(item)
print "#"*30
it=item*2
print(it)
t3.join()
t4.join()
def prints(self,i):
while(i>0):
print "i="+str(i)+" "+str(datetime.datetime.now().time())+"\n"
i=i-1
def printy(self,i,b):
r=0
while(i<10):
print "i="+str(i)+" "+str(datetime.datetime.now().time())+"\n"
i=i+1
r=r+i
self.que.put(r)
if __name__=='__main__':
MyThread(a).main()
However, even though I manage to get the output as I want. I'm curious to know that how the same can be achieved if my function printy() is modified as below:
def printy(self,i,b):
r=0
while(i<10):
print "i="+str(i)+" "+str(datetime.datetime.now().time())+"\n"
i=i+1
r=r+i
return(r)
How could queue be used to get the returned values? Or do we have any alternate way to do this?
You only ever put items in the queue but never get them out. At the end of __init__, however, you join the queue, which blocks until the queue has been consumed. Since up until that point, no consumer has been launched, it will block forever.
Update
I'm not sure what you're trying to achieve: now your program will terminate with the queue still active in the background; in both __init__ and main, you put in 4 items but get out only one. If that really produces the output you want, I'm not sure what to suggest.
On the other hand, assuming that you're trying to implement a producer-consumer system, you could something like this:
import Queue
import threading
class ProduceConsume(object):
def produce(self, n):
for i in xrange(n):
print 'producing {}'.format(i)
self.queue.put(i)
def consume(self):
while self.running:
try:
i = self.queue.get(timeout=0.1)
except Queue.Empty:
continue
print 'consuming {}'.format(i)
self.queue.task_done()
def main(self, n):
self.queue = Queue.Queue()
self.running = True
consumer = threading.Thread(target=self.consume)
consumer.start()
producer = threading.Thread(target=self.produce, args=(n,))
producer.start()
producer.join()
self.queue.join()
self.running = False
consumer.join()
ProduceConsume().main(4)
Now, if you want to obtain a kind of result value from the producer thread (the one that puts items in the queue like your printy, you cannot simply return it since the return value of a thread's execution target isn't used. You could store it somewhere (maybe as an attribute on your ProduceConsume instance) and access it after the producer thread has been joined. Since you ask how to use the queue for it, however, you'd need to put it in the queue as the last item and handle it appropriately in the consumer thread:
def produce(self, n):
r = 0
for i in xrange(n):
print 'producing {}'.format(i)
self.queue.put(i)
r += i
self.queue.put(r)
def consume(self):
while self.running:
try:
i = self.queue.get(timeout=0.1)
except Queue.Empty:
continue
print 'consuming {}'.format(i)
self.queue.task_done()
else:
print 'result was: {}'.format(i)
To further avoid treating the result as just another item before finding that it was the last item gotten from the queue, you'll need more complex items, such as ('item', i) and ('result', r) that tell you whether they represent a regular item or the result.
I'm trying to understand the basics of threading and concurrency. I want a simple case where two threads repeatedly try to access one shared resource.
The code:
import threading
class Thread(threading.Thread):
def __init__(self, t, *args):
threading.Thread.__init__(self, target=t, args=args)
self.start()
count = 0
lock = threading.Lock()
def increment():
global count
lock.acquire()
try:
count += 1
finally:
lock.release()
def bye():
while True:
increment()
def hello_there():
while True:
increment()
def main():
hello = Thread(hello_there)
goodbye = Thread(bye)
while True:
print count
if __name__ == '__main__':
main()
So, I have two threads, both trying to increment the counter. I thought that if thread 'A' called increment(), the lock would be established, preventing 'B' from accessing until 'A' has released.
Running the makes it clear that this is not the case. You get all of the random data race-ish increments.
How exactly is the lock object used?
Additionally, I've tried putting the locks inside of the thread functions, but still no luck.
You can see that your locks are pretty much working as you are using them, if you slow down the process and make them block a bit more. You had the right idea, where you surround critical pieces of code with the lock. Here is a small adjustment to your example to show you how each waits on the other to release the lock.
import threading
import time
import inspect
class Thread(threading.Thread):
def __init__(self, t, *args):
threading.Thread.__init__(self, target=t, args=args)
self.start()
count = 0
lock = threading.Lock()
def incre():
global count
caller = inspect.getouterframes(inspect.currentframe())[1][3]
print "Inside %s()" % caller
print "Acquiring lock"
with lock:
print "Lock Acquired"
count += 1
time.sleep(2)
def bye():
while count < 5:
incre()
def hello_there():
while count < 5:
incre()
def main():
hello = Thread(hello_there)
goodbye = Thread(bye)
if __name__ == '__main__':
main()
Sample output:
...
Inside hello_there()
Acquiring lock
Lock Acquired
Inside bye()
Acquiring lock
Lock Acquired
...
import threading
# global variable x
x = 0
def increment():
"""
function to increment global variable x
"""
global x
x += 1
def thread_task():
"""
task for thread
calls increment function 100000 times.
"""
for _ in range(100000):
increment()
def main_task():
global x
# setting global variable x as 0
x = 0
# creating threads
t1 = threading.Thread(target=thread_task)
t2 = threading.Thread(target=thread_task)
# start threads
t1.start()
t2.start()
# wait until threads finish their job
t1.join()
t2.join()
if __name__ == "__main__":
for i in range(10):
main_task()
print("Iteration {0}: x = {1}".format(i,x))