I know there's something stupid I'm doing, but I can't figure this out! I'm testing by doing 1 thread. It's supposed to add up to ten items in the queue, wait for it to finish, and then say "Finished". The thread function has q.task_done() at the end of it. But it'll say "Task Done" but then it just pauses and doesn't ever say "Finished".
def helper(q):
print("Started")
....
q.task_done()
print("Task done")
def main():
q = Queue(maxsize=0)
recid = 9000000
num_threads = 1
for i in range(num_threads):
worker = Thread(target=helper, args=(q))
worker.setDaemon(True)
worker.start()
while recid > 0:
batch.clear()
for x in range(10):
q.put(recid)
print("Added request.")
recid -= 1
print("Waiting for threads to finish.")
q.join()
print("Finished")
worker was dead before you called q.join(). maybe try to add some delays in helper() waiting for q.put()
def helper(q):
print("Started")
time.sleep(1) # wait for main() q.put()
while not q.empty(): # until queue is empty
print("request", q.get())
q.task_done() # trigger q.join in main()
print("Task done")
time.sleep(1) # wait for next q.put() if exists
Updated:
Basically, q.task_done() in helper() triggers q.join() in main() and worker.join() will be triggered once helper() is exited. An example code is as followed:
def main():
q = Queue(maxsize=0)
recid = 100
num_threads = 4
while recid > 0:
for i in range(num_threads):
print('''******** Hired Worker %d ********'''%(i),)
worker = Thread(target=helper, args=(q,))
worker.setDaemon(True)
worker.start()
# batch.clear()
for x in range(3):
q.put(recid)
print("Request Worker %d to do %d."%(i, recid))
recid -= 1
##############################################################
# Comment these lines when you are ready to test multiThread
##############################################################
print("Waiting Worker %d to finish %d."%(i, recid))
q.join() # triggered by q.task_done in helper()
print("Worker %d Finished %d"%(i, recid))
print("Waiting for Worker %d to leave."%i)
worker.join() # triggered once helper() is exited
print("!!!!!!!! Worker %d left !!!!!!!!"%i)
##############################################################
time.sleep(5) # wait until all requests are finished.
print("Finished")
Once you are happy with join() of Thread and Queue. You may try multithreading by commenting that block of code.
Related
I want to measure how much time it takes to finish running the code with multiple threads in python.
If I put join inside the loop, it will stop the loop (main thread) from keep creating new threads. It will run the sleep() one by one.
If I put join on the thread which I use to create thread_testing, the join won't work somehow. It prints out the time immediately.
def sleep(name):
print("{} going to sleep".format(name))
time.sleep(5)
print("{} wakes up after 5 seconds".format(name))
def thread_testing():
for i in range(3):
t = threading.Thread(target=sleep, name='thread' + str(i), args=(i,)
t.start()
# t.join() #1
if __name__ == '__main__':
start = time.time()
t = threading.Thread(target=thread_testing, name='threadx')
t.start()
t.join() #2
print(time.time() - start)
Desired output:
1 sleep
2 sleep
3 sleep
1 wake up after 5
2 wake up after 5
3 wake up after 5
5.xxx secs
Join will wait for your thread. That is why your threads were executed one by one.
What you have to do is:
Start all threads
Store them somewhere
Once everything is started wait for every thread to finish.
Assuming you don't need the first thread started in main:
import time
import threading
def sleep(name):
print("{} going to sleep".format(name))
time.sleep(5)
print("{} wakes up after 5 seconds".format(name))
def thread_testing():
threads = []
for i in range(3):
t = threading.Thread(target=sleep, name='thread' + str(i), args=(i,))
t.start()
threads.append(t)
for t in threads:
t.join()
if __name__ == '__main__':
start = time.time()
thread_testing()
print(time.time() - start)
why this code is not running in parallel, it take 20 seconds to run it, which means it is running in sequence. Thank you for your help.
import time
from queue import Queue
from threading import Thread
start = time.time()
def f():
time.sleep(0.5)
print("yes")
return 'yes'
def do_stuff(q):
while True:
output = q.get()
q.task_done()
q = Queue(maxsize=100)
for message_nbr in range(40):
q.put(f())
num_threads = 10
for i in range(num_threads):
worker = Thread(target=do_stuff, args=(q, ))
worker.setDaemon(True)
worker.start()
q.join()
print("time: ", time.time() - start) # ~20 seconds
The answer lies here:
for message_nbr in range(40):
q.put(f())
You're putting 40 instances of None into your queue, because you're calling f() which returns None instead of passing f (the function object) in. This block is what is taking 20 seconds to run!
Changing this code
def do_stuff(q):
while True:
output = q.get()
q.task_done()
to this
def do_stuff(q):
while True:
output = q.get()
output()
q.task_done()
is also necessary (you need to call the function!)
Final:
import time
from queue import Queue
from threading import Thread
start = time.time()
def f():
time.sleep(0.5)
print("yes")
return 'yes'
def do_stuff(q):
while True:
output = q.get()
output()
q.task_done()
q = Queue(maxsize=100)
for message_nbr in range(40):
q.put(f)
num_threads = 10
for i in range(num_threads):
worker = Thread(target=do_stuff, args=(q, ))
worker.setDaemon(True)
worker.start()
q.join()
print("time: ", time.time() - start) # time: 2.183439254760742
this one works !
start = time.time()
def f(m):
time.sleep(0.5)
print("yes")
return 'yes'
def do_stuff(q):
while True:
output = q.get()
final_result = f(output)
q.task_done()
q = Queue(maxsize=0)
for message_nbr in range(10):
# q.put(f())
q.put(message_nbr)
num_threads = 10
for i in range(num_threads):
worker = Thread(target=do_stuff, args=(q, ))
worker.setDaemon(True)
worker.start()
q.join()
print("time: ", time.time() - start)
I would like to exit the program gracefully on Ctrl+C / SIGINT or on user input. If possible the terminal should prompt something like; "Hit enter to terminate".
Code to be executed by Python 3.6
def worker(process):
i = 0
while True:
print('Process %d count %d' % (process, i))
i += 1
def main():
available_num_cores = multiprocessing.cpu_count()
use_num_cores = available_num_cores - 1 if available_num_cores > 1 else 1
print('Using %d cores' % use_num_cores)
pool = multiprocessing.Pool(use_num_cores)
for i in range(0, use_num_cores):
pool.apply_async(worker, args=(i,))
pool.close()
pool.join()
if __name__ == '__main__':
main()
Accepted answer for this question Catch Ctrl+C / SIGINT and exit multiprocesses gracefully in python. Isn't working, it fail with error:
Process SpawnPoolWorker-1:
Process 0 count 1572
Process SpawnPoolWorker-2:
Process 1 count 1472
Process SpawnPoolWorker-3:
Traceback (most recent call last):
Any help would be appreciated. Thanks!
You need to make sure the SIGINT is ignored by the children processes.
Then you just either wait for user input or for a CTRL+C to be issued.
def initializer():
"""Ignore SIGINT in child workers."""
signal.signal(signal.SIGINT, signal.SIG_IGN)
def main():
try:
pool = multiprocessing.Pool(use_num_cores, initializer=initializer)
for i in range(0, use_num_cores):
pool.apply_async(worker, args=(i,))
pool.close()
input("Hit enter to terminate")
except KeyboardInterrupt:
print("CTRL+C")
finally:
pool.terminate()
pool.join()
print("Bye have a great time!")
Consider the following example I've been doing to learn multithreading. It's just an extended example of the Python 3.5 queue documentation.
It prints some numbers over 4 threads, produces one error in the queue, retries this element and should print the remaining queue if a KeyboardInterrupt exception occurs.
import threading
import queue
import time
import random
import traceback
def worker(q, active):
while True:
worker_item = q.get()
#if worker_item == None:
if not active.is_set():
break
time.sleep(random.random())
with threading.Lock():
if worker_item == 5 or worker_item == '5':
try:
print(threading.current_thread().name + ': ' + worker_item + ' | remaining queue: ' + str(list(q.queue)))
except TypeError:
print(threading.current_thread().name + ': ')
print(traceback.format_exc())
q.put(str(worker_item))
else:
print(threading.current_thread().name + ': ' + str(worker_item) + ' | remaining queue: ' + str(list(q.queue)))
q.task_done()
def main():
# INITIALIZE
num_threads = 4
stack1 = list(range(1, 21))
stack2 = list(range(101, 121))
q = queue.Queue()
active = threading.Event()
active.set()
# START THREADS
threads = []
for _ in range(num_threads):
t = threading.Thread(target=worker, args=(q, active))
t.start()
threads.append(t)
try:
# PUT STACK ITEMS ON QUEUE AND BLOCK UNTIL ALL TASKS ARE DONE
for stack1_item in stack1:
q.put(stack1_item)
q.join()
for stack2_item in stack2:
q.put(stack2_item)
q.join()
# STOP WORKER LOOP IN EVERY THREAD
#for _ in threads:
#q.put(None)
active.clear()
# WAIT UNTIL ALL THREADS TERMINATE
for t in threads:
t.join()
except KeyboardInterrupt:
print(traceback.format_exc())
print('remaining queue: ' + str(list(q.queue)))
#for _ in threads:
#q.put(None)
active.clear()
for t in threads:
t.join()
if __name__ == '__main__':
main()
If I run the script as it is (without a KeyboardInterrupt), it won't terminate. I have to kill the signal. But if I comment/uncomment the following lines (not using the event and doing it the docs way...)
comment / worker / if not active.is_set():
uncomment / worker / #if worker_item == None:
comment / main / active.clear()
uncomment / main / #for _ in threads:
#q.put(None)
comment / main / except / active.clear()
uncomment / main / except / #for _ in threads:
#q.put(None)
it does exit with exit code 0. Why?
Why is putting Nones to the queue necessary?
What would be the solution without putting Nones to the queue?
There are two types of threads: daemon and non-daemon. By default, all threads are non-daemon. The process is kept alive as long as there is at least one non-daemon thread.
This means that to stop the process, you either have to:
stop all of its threads (this is what your commented out code does by using None to kick the worker out of the infinite wait in q.get()); or
make the workers daemon threads, in which case the process will stop as soon as the main thread stops (this will require extra care if you want to ensure the workers have finished their tasks).
I am new with python and I am trying to made code that stop work in case timeout has been reached. But it seems that the tread correctly timeout but it does not stop the work.
Here is my code:
import threading
import time
import sys
def main():
t1_stop= threading.Event()
t1 = threading.Thread(target=thread1, args=(5, t1_stop))
t1.setDaemon(False)
t1.start()
print 'thread 1 set'
while True:
print "blablabla"
time.sleep(1)
def thread1(time, stop_event):
while(not stop_event.is_set()):
#equivalent to time.sleep()
print 'thread 1'
stop_event.wait(time)
main()
UPD
I update code using Timer instead of time.time.
def main():
stopped = threading.Event()
timeout = 10
#thread = threading.Thread(target=my_thread, args=(timeout, stopped))
timer = Timer(timeout, my_thread(timeout,stopped))
thread = threading.Thread((timer).start())
thread.setDaemon(False)
#thread.start()
print 'thread 1 set'
start_t = time.time()
while thread.is_alive():
print "doing my job"
if not stopped.is_set():# and (time.time() - start_t) > timeout:
stopped.set()
#time.sleep(1)
def my_thread(time, stopped):
while not stopped.wait(time):
print('thread stopped')
main()
But I still get original problem the script does not stopped and continue.
Thanks in advance for your help.
you must call t1_stop.set() in main function for stop thread.
something like:
import threading
import time
import sys
def main():
stopped = threading.Event()
thread = threading.Thread(target=my_thread, args=(5, stopped))
thread.setDaemon(False)
thread.start()
print 'thread 1 set'
time.sleep(5) # +
stopped.set() # +
while True:
print "blablabla"
time.sleep(1)
def my_thread(time, stopped):
while not stopped.wait(time):
print('thread 1')
main()
with 'blablabla':
def main():
...
thread.start()
print 'thread 1 set'
start_t = time.time()
while True:
print "blablabla"
if not stopped.is_set() and (time.time() - start_t) > 5:
stopped.set()
time.sleep(1)
UPD:
exiting from while:
while thread.is_alive():
print "blablabla"
if not stopped.is_set() and (time.time() - start_t) > 5:
stopped.set()
time.sleep(1)