Develop Reference

How to exit a while loop in a multiprocessing pool process from the main script in python

EDIT 3: See last example at the end.
I need a while loop doing continuous send and return operations with an USB connection.
During this continuous operation I need (amongst other stuff in my main script) a few identical and isolated send/return operations on that same USB connection.
This seems to require multiprocessing and some tweaking.
I want to use the following workaround with the multiprocessing library:
Put the continuous send/return operation on a different thread with a pool (apply_async).
Put this process on "hold" when I perform the isolated send/return operation (using clear()).
Immediately after the isolated send/return operation resume the continuous send/return (using set()).
Stop the continuous send/return when i reach the end of the main script (here i have no solution yet should be x.stop() or something like this since terminate() won't do).
Get some return value from the stopped process (use get()).
I tried couple of things already but i just cant exit the while loop via a main command.
import multiprocessing
import time
def setup(event):
global unpaused
unpaused = event
class func:
def __init__(self):
self.finished = False
def stop(self):
self.finished = True
def myFunction(self, arg):
i = 0
s=[]
while self.finished == False:
unpaused.wait()
print(arg+i)
s.append(arg+i)
i=i+1
time.sleep(1)
return s
if __name__ == "__main__":
x=func()
event = multiprocessing.Event() # initially unset, so workers will be paused at first
pool = multiprocessing.Pool(1, setup, (event,))
result = pool.apply_async(x.myFunction, (10,))
print('We unpause for 2 sec')
event.set() # unpause
time.sleep(2)
print('We pause for 2 sec')
event.clear() # pause
time.sleep(2)
print('We unpause for 2 sec')
event.set() # unpause
time.sleep(2)
print('Now we try to terminate in 2 sec')
time.sleep(2)
x.stop()
return_val = result.get()
print('get worked with '+str(return_val))
Can someone point me in the right direction? As seen this wont stop with x.stop().
Global values also do not work.
Thanks in advance.
EDIT:
as suggested I tried to put the multiprocessing in a seperated object.
Is this done by putting functions in a class like my example below?
import multiprocessing
import time
class func(object):
def __init__(self):
self.event = multiprocessing.Event() # initially unset, so workers will be paused at first
self.pool = multiprocessing.Pool(1, self.setup, (self.event,))
def setup(self):
global unpaused
unpaused = self.event
def stop(self):
self.finished = True
def resume(self):
self.event.set() # unpause
def hold(self):
self.event.clear() #pause
def run(self, arg):
self.pool.apply_async(self.myFunction, (arg,))
def myFunction(self, arg):
i = 0
s=[]
self.finished = False
while self.finished == False:
unpaused.wait()
print(arg+i)
s.append(arg+i)
i=i+1
time.sleep(1)
return s
if __name__ == "__main__":
x=func()
result = x.run(10)
print('We unpause for 2 sec')
x.resume() # unpause
time.sleep(2)
print('We pause for 2 sec')
x.hold() # pause
time.sleep(2)
print('We unpause for 2 sec')
x.resume() # unpause
time.sleep(2)
print('Now we try to terminate in 2 sec')
time.sleep(2)
x.stop()
return_val = result.get()
print('get worked with '+str(return_val))
I added a hold and resume function and put the setup function in a single class.
But the lower example wont even run the function anymore.
What a complex little problem. I am puzzled with this.
EDIT2:
I tried a workaround with what i found so far.
Big trouble came in while using the microprocessing.pool library.
It is not straightforward using it with the USB connection...
I produced a mediocre workaround below:
from multiprocessing.pool import ThreadPool
import time
class switch:
state = 1
s1 = switch()
def myFunction(arg):
i = 0
while s1.state == 1 or s1.state == 2 or s1.state == 3:
if s1.state == 1:
print(arg+i)
s.append(arg+i)
i=i+1
time.sleep(1)
elif s1.state == 2:
print('we entered snippet mode (state 2)')
time.sleep(1)
x = s
return x
pool.close()
pool.join()
elif s1.state == 3:
while s1.state == 3:
time.sleep(1)
print('holding (state 3)')
return s
if __name__ == "__main__":
global s
s=[]
print('we set the state in the class on top to ' +str(s1.state))
pool = ThreadPool(processes=1)
async_result = pool.apply_async(myFunction, (10,))
print('in 5 sec we switch mode sir, buckle up')
time.sleep(5)
s1.state = 2
print('we switched for a snippet which is')
snippet = async_result.get()
print(str(snippet[-1])+' this snippet comes from main')
time.sleep(1)
print('now we return to see the full list in the end')
s1.state = 1
async_result = pool.apply_async(myFunction, (10,))
print('in 5 sec we hold it')
time.sleep(5)
s1.state = 3
print('in 5 sec we exit')
time.sleep(5)
s1.state = 0
return_val = async_result.get()
print('Succsses if you see a list of numbers '+ str(return_val))
EDIT 3:
from multiprocessing.pool import ThreadPool
import time
class switch:
state = 1
s1 = switch()
def myFunction(arg):
i = 0
while s1.state == 1 or s1.state == 2:
if s1.state == 1:
print(arg+i)
s.append(arg+i)
i=i+1
time.sleep(1)
elif s1.state == 2:
print('we entered snippet mode (state 2)')
time.sleep(1)
x = s
return x
pool.close() #These are not relevant i guess.
pool.join() #These are not relevant i guess.
return s
if __name__ == "__main__":
global s
s=[]
print('we set the state in the class on top to ' +str(s1.state))
pool = ThreadPool(processes=1)
async_result = pool.apply_async(myFunction, (10,))
print('in 5 sec we switch mode sir, buckle up')
time.sleep(5)
s1.state = 2
snippet = async_result.get()
print(str(snippet[-1])+' this snippet comes from main')
time.sleep(1)
print('now we return to see the full list in the end')
s1.state = 1
async_result = pool.apply_async(myFunction, (10,))
print('in 5 sec we exit')
time.sleep(5)
s1.state = 0
return_val = async_result.get()
print('Succsses if you see a list of numbers '+ str(return_val))
Well, this is what i have come up with...
Not great not terrible. Maybe a bit more on the terrible side (:
I hate it that I have to recall the function pool.apply_async(myFunction, (10,)) after I grabbed a single piece of data.
Currently only ThreadingPool works with no further code changes in my actual script!

in a situation where I need a process to run continuously, while occasionally doing other things, I like to use asyncio. This is a rough draft of how I would approach this
import asyncio
class MyObject:
def __init__(self):
self.mydatastructure = []
self.finished = False
self.loop = None
async def main_loop(self):
while not self.finished:
new_result = self.get_data()
self.mydatastructure.append(new_result)
await asyncio.sleep(0)
async def timed_loop(self):
while not self.finished:
await asyncio.sleep(2)
self.dotimedtask(self.mydatastructure)
async def run(self):
await asyncio.gather(self.main_loop(), self.timed_loop())
asyncio.run(MyObject().run())
only one coroutine will be running at a time, with the timed one being scheduled once every 2 seconds. It would always get the data passed out of the most recent continuous execution. you could do things like keep a connection open on an object as well. Depending on your requirements (is it a 2 second interval, or once every other second no matter how long it takes) there are library packages to make the scheduling a bit more elegant.

Blocking other Threads while one Thread is running

Let's say I have two types of threads,
single thread that run every x min. let's call it A thread
multi threads run all the time. B threads when A thread do_something() i want All B threads to wait till A finish then resume them. i can't figure it out what to use.
I try to use threading.Condition, wait()/notifyAll() but it did not work as I want. once i put Condition in, it process 1 by 1 like synco threads or something. I want them to run freely.
This is the sample code I try to put them wait(), then notify them but it do 1 by 1 like join(). No idea what to us.
class ...
check = True
def xxx(self,g,con):
for i in range(3):
with con:
if self.check:
con.wait()
self.check = False
time.sleep(3)
print(g)
con = threading.Condition()
threading.Thread(target=xxx,args=('a',con,)).start()
threading.Thread(target=xxx,args=('b',con,)).start()
threading.Thread(target=xxx,args=('c',con,)).start()
time.sleep(2)
con.notifyAll()

Question: Blocking other Threads while one Thread is running
Instead of using threading.Condition(), this example uses threading.Barrier(...).
Used modules from docs.python.org:
module-threading
event-objects
barrier-objects
import time, threading
from threading import BrokenBarrierError
def worker_A(g, terminate, barrier):
# Counter to simulate conditional workload
do_something = 3
while not terminate.is_set():
if do_something == 0:
# Reset the barrier and wait until n_waiting == 2
barrier.reset()
while not terminate.is_set() and barrier.n_waiting < 2:
time.sleep(0.5)
# Now the other Threads waiting at the barrier
# Simulate worklaod ...
print('worker_A barrier.broken={} n_waiting={}'
.format(barrier.broken, barrier.n_waiting))
time.sleep(3)
# Call the third barrier.wait to release the barrier
try:
barrier.wait()
except BrokenBarrierError:
pass
# Reset counter to restart simulate conditional workload
do_something = 3
else:
# Count down and give the other threads a timeslice
do_something -= 1
time.sleep(0.5)
def worker_B(g, terminate, barrier):
while not terminate.is_set():
# Simulate workload ...
print('worker_B({})'.format(g))
time.sleep(1)
# Block at barrier.wait() if the barrier is NOT in the broken state
try:
barrier.wait()
except BrokenBarrierError:
pass
if __name__ == "__main__":
# Event to terminate all Threads save
terminate = threading.Event()
# Barrier to block worker_B Threads
# We use 3 Threads, therefore init with parties=3
barrier = threading.Barrier(3)
barrier.abort()
# Create and start the Threads
threads = []
for t in [(worker_A, 'a'), (worker_B, 'b'), (worker_B, 'c'), ]:
threads.append(threading.Thread(target=t[0], args=(t[1], terminate, barrier,)))
threads[-1].start()
time.sleep(0.2)
# Simulating MAIN Thread
time.sleep(20)
# Set the `terminate` Event to True,
# and abort the barrier to force all Threads to terminate
print('Terminate...')
terminate.set()
barrier.abort()
# Wait until all Threads terminated
for t in threads:
t.join()
print('EXIT MAIN')
Tested with Python: 3.5

Run while loop until function has returned a value

I'm trying to light a 5mm LED while a function is running. When this function (more details about this below) is finished and has returned a value I would like to break the while loop.
Current code for while loop:
pins = [3,5,8,15,16]
def piBoard():
finished = 0
while finished!=10:
for pin in pins
GPIO.output(
pin, GPIO.HIGH
)
time.sleep(0.1)
GPIO.output(
pin, GPIO.LOW
)
finished+=1
Now in the above example I just run the while loop until the count is equal to 10, not best practice. I would like the while loop to break if my next function has returned a value.
Function I want to break my while loop when returned its value
def myFunction():
Thread(target = piBoard().start()
// Trying to recognize the song
return the song which is recognized
Thanks, - K.

It sounds to me like you want to write a class that extends Thread and implements __enter__ and __exit__ methods to make it work in the with statement. Simple to implement, simple syntax, works pretty well. The class will look like this:
import threading
class Blinky(threading.Thread):
def __init__(self):
super().__init__()
self.daemon = True
self._finished = False
def __enter__(self):
self.start()
def __exit__(self, exc_type, exc_val, exc_tb):
self.stop()
def run(self):
# turn light on
while not self._finished:
time.sleep(.5)
# turn light off
def stop(self):
self._finished = True
Then, to run your function, you simply put:
with Blinky():
my_function()
The light should turn on once the with statement is reached and turn off up to a half second after the context of the with is exited.

In while condition put true and in while loop put if statement which will check if your function return any value if return write break

You need some kind of inter-thread communication. threading.Event is about as simple as you can get.
import threading
song_recognized_event = threading.event()
in your song recognizer, call set() once the song is recognized.
In your LED loop, check isSet() occasionally while toggling LEDs.
while not song_recognized_event.isSet():
# toggle LEDs
Run clear() to reset it.

if you are open to using threads.
you can achieve this by using threads.
here's the example code
from concurrent.futures._base import as_completed
from concurrent.futures.thread import ThreadPoolExecutor
WORK_FINISHED = False
def piBoard():
while not WORK_FINISHED:
# Do some stuff
# Drink some coffee
def myFunction():
time.sleep(5)
global WORK_FINISHED
WORK_FINISHED = True #update gobal status flag
return something
if __name__ == '__main__':
futures = []
MAX_WORKERS = 5 #max number of threads you want to create
with ThreadPoolExecutor(MAX_WORKERS) as executor:
executor.submit(piBoard)
# submit your function to worker thread
futures.append(executor.submit(myFunction))
# if you need to get return value from `myFunction`
for fut in as_completed(futures):
res = fut.result()
Hope this helps.

Using decorator and asyncio, inspired by #Eric Ed Lohmar:
import asyncio
def Blink():
from functools import wraps
async def _blink():
while True:
print("OFF")
await asyncio.sleep(.5)
print("ON")
await asyncio.sleep(.5)
def Blink_decorator(func):
#wraps(func)
async def wrapper(*args,**kwargs):
asyncio.ensure_future(_blink())
await func(*args,**kwargs)
return wrapper
return Blink_decorator
#Blink()
async def longTask():
print("Mission Start")
await asyncio.sleep(3)
print("Mission End")
def main():
loop = asyncio.get_event_loop()
loop.run_until_complete(longTask())

How to use a thread pool to do infinite loop function?

I want to do a infinite loop function.
Here is my code
def do_request():
# my code here
print(result)
while True:
do_request()
When use while True to do this, it's a little slow, so I want to use a thread pool to concurrently execute the function do_request(). How to do this ?
Just like use ab (Apache Bench) to test HTTP server.

Finally, I've solved this problem. I use a variable to limit the thread number.
Here is my final code, solved my problem.
import threading
import time
thread_num = 0
lock = threading.Lock()
def do_request():
global thread_num
# -------------
# my code here
# -------------
with lock:
thread_num -= 1
while True:
if thread_num <= 50:
with lock:
thread_num += 1
t = threading.Thread(target=do_request)
t.start()
else:
time.sleep(0.01)
Thanks for all replies.

You can use threading in Python to implement this.
Can be something similar to this (when using two extra threads only):
import threading
# define threads
task1 = threading.Thread(target = do_request)
task2 = threading.Thread(target = do_request)
# start both threads
task1.start()
task2.start()
# wait for threads to complete
task1.join()
task2.join()
Basically, you start as many threads as you need (make sure you don't get too many, so your system can handle it), then you .join() them to wait for tasks to complete.
Or you can get fancier with multiprocessing Python module.

Try the following code:
import multiprocessing as mp
import time
def do_request():
while(True):
print('I\'m making requests')
time.sleep(0.5)
p = mp.Process(target=do_request)
p.start()
for ii in range(10):
print 'I\'m also doing other things though'
time.sleep(0.7)
print 'Now it is time to kill the service thread'
p.terminate()
The main thread stars a service thread that does the request and goes on until it has to, and then it finishes up the service thread.

Maybe you can use the concurrent.futures.ThreadPoolExecutor
from concurrent.futures import ThreadPoolExecutor
import time
def wait_on_b(hello):
time.sleep(1)
print(hello) # b will never complete because it is waiting on a.
return 5
def wait_on_a():
time.sleep(1)
print(a.result()) # a will never complete because it is waiting on b.
return 6
executor = ThreadPoolExecutor(max_workers=2)
a = executor.submit(wait_on_b, 3)
b = executor.submit(wait_on_a)

How about this?
from threading import Thread, Event
class WorkerThread(Thread):
def __init__(self, logger, func):
Thread.__init__(self)
self.stop_event = Event()
self.logger = logger
self.func = func
def run(self):
self.logger("Going to start the infinite loop...")
#Your code
self.func()
concur_task = WorkerThread(logger, func = do_request)
concur_task.start()
To end this thread...
concur_task.stop_event.set()
concur_task.join(10) #or any value you like

Python 3, multithreading - getting data into main module as each thread finishes

The following code executes two threads (multithread), each with different time delays so that each thread will finish at a different time.
Once both threads are finished module display1.py issues a print statement saying they are BOTH finished.
I would like module display1.py to issue a 'finished' statement for EACH thread AS EACH thread finishes
How can i do this ... amendments to my working code appreciated! I'd like to change as little of the current code as possible so a better form of variable transfer between the two modules might be what I'm after
display1.py
from threads1 import *
manager = ThreadManager()
manager.start(False)
print (manager.GetResults())
threads1.py
from threading import Thread
import time
class ThreadManager:
def __init__(self):
pass
def start(self, answer):
self.answer = answer
thread_refs = []
t1 = MyThread(70, 'Not finished')
t1.daemon = True
t1.start()
t2 = MyThread(2, 'Not finished')
t2.daemon = True
t2.start()
while True:
if t1.AskFinished == 'Finished' and t2.AskFinished == 'Finished': #If I break the loop after EACH site, Only the first to finish will be sent via GetResults to display1.py
global results
results = [t1.AskFinished, t2.AskFinished]
print("Both Finished")
break
def GetResults(self):
global results
return(results)
class MyThread(Thread):
def __init__(self, SleepWait, AskFinished):
Thread.__init__(self)
self.SleepWait = SleepWait
self.AskFinished = AskFinished
def run(self):
time.sleep(self.SleepWait)
self.AskFinished = 'Finished'

What you have here (entering a very tight check loop in the main thread) is a very naive approach to threading in many languages, but especially in python where GIL contention will just slow the threads down a great bit.
What is a better idea is instead using queue.Queue to push info when a thread is completed. This allows the main thread to block on the queue instead, which is less CPU intensive as well as allowing you to know (out of order) which one is finished.
The changes you would need to make:
at the top of the module threads1.py:
import queue
finished_queue = queue.Queue()
in your start():
num_finished = 0
while True:
info = finished_queue.get()
num_finished += 1
if info is t1:
print("t1 finished")
else:
print("t2 finished")
if num_finished == 2:
global results
results = [t1.AskFinished, t2.AskFinished]
print("Both Finished")
break
and finally in run():
def run(self):
time.sleep(self.SleepWait)
self.AskFinished = 'Finished'
finished_queue.put(self)
Some more fundamental modifications I'd make is actually pushing the result into the queue and then fetching the results out, skipping the extra step before GetResults. Furthermore, if GetResults had to stay, I'd pass them through a field on self e.g. self.results = [t1.AskFinished, t2.AskFinished]

Update:
Ok, so you want to know more about how to have display1.py print the results. It would be helpful if you could explain why it matters, because that might make a difference in how you should do this, but here's a first approach:
# threads1.py
from threading import Thread
import time
class ThreadManager:
def __init__(self):
self.threads = {}
def start(self):
t1 = MyThread(4)
t1.daemon = True
t1.start()
self.threads[1] = t1
t2 = MyThread(1)
t2.daemon = True
t2.start()
self.threads[2] = t2
def is_alive(self, thread_id):
return self.threads[thread_id].is_alive()
def GetResults(self): # or you could just access results directly
return self.results
class MyThread(Thread):
def __init__(self, SleepWait):
Thread.__init__(self)
self.SleepWait = SleepWait
def run(self):
time.sleep(self.SleepWait)
And then...
# display1.py
from threads1 import *
manager = ThreadManager()
manager.start()
t1_state = t2_state = True
while manager.is_alive(1) or manager.is_alive(2):
time.sleep(1)
if manager.is_alive(1) != t1_state:
print("t1 finished")
t1_state = manager.is_alive(1)
if manager.is_alive(2) != t2_state:
print("t2 finished")
t2_state = manager.is_alive(2)
if not manager.is_alive(1) and not manager.is_alive(2):
print("Both Finished")
break
You should eventually consider using a Queue as suggested by Crast; but let's focus on getting this right first.
Original Post:
There are a number of problems with this code.
First, you should use t1.is_alive() to check if a thread is finished. There's no need to reimplement it with AskFinished.
Second, the while True: loop in threads1.py is doing nothing at a furious rate while it waits for your threads to terminate. Take a look at the cpu usage while this is running if you don't believe me. You should throw a time.sleep(1) statement in there.
Third, why are you using a global var to return your results? That's a really strange thing to do. Just store it in self!
And finally, why does display1.py have to print the messages? Why can't thread1.py do that?
With these four points in mind, here's a thread1.py that works more sensibly:
from threading import Thread
import time
class ThreadManager:
def __init__(self):
self.results = None
def start(self, answer): # why is "answer" here?
self.answer = answer
thread_refs = []
t1 = MyThread(4, 'Not finished')
t1.daemon = True
t1.start()
t2 = MyThread(1, 'Not finished')
t2.daemon = True
t2.start()
t1_state = t2_state = True
while t1.is_alive() or t2.is_alive():
time.sleep(1)
if t1.is_alive() != t1_state:
print("t1 finished")
t1_state = t1.is_alive()
if t2.is_alive() != t2_state:
print("t2 finished")
t2_state = t2.is_alive()
if not t1.is_alive() and not t2.is_alive():
self.results = [t1.AskFinished, t2.AskFinished]
print("Both Finished")
break
def GetResults(self): # or you could just access results directly
return self.results
class MyThread(Thread):
def __init__(self, SleepWait, AskFinished):
Thread.__init__(self)
self.SleepWait = SleepWait
self.AskFinished = AskFinished
def run(self):
time.sleep(self.SleepWait)
self.AskFinished = 'Finished'
Now, this still doesn't do exactly what you wanted, because you asked for display.py to do the displaying. To make that work, you'd have to put your while True loop in display.py and add an ThreadManager.is_alive() method that display.py could use to check whether a thread is alive or not. If you want to see how to do that let me know.

Im not familiar with threading but since no answers yet ill give it a shot.
In this:
Cant you just add two if statements before hand?
while True:
if t1.askFinished == 'Finished':
print("t1 Finished")
if t2.askFinished == 'Finished':
print("t2 Finished")
if t1.AskFinished == 'Finished' and t2.AskFinished == 'Finished': #If I break the loop after EACH site, Only the first to finish will be sent via GetResults to display1.py
global results
results = [t1.AskFinished, t2.AskFinished]
print("Both Finished")
break
edit: I tried changing your code as little as possible... it's not very well written though tbh. (No offense)