why this code is not running in parallel, it take 20 seconds to run it, which means it is running in sequence. Thank you for your help.
import time
from queue import Queue
from threading import Thread
start = time.time()
def f():
time.sleep(0.5)
print("yes")
return 'yes'
def do_stuff(q):
while True:
output = q.get()
q.task_done()
q = Queue(maxsize=100)
for message_nbr in range(40):
q.put(f())
num_threads = 10
for i in range(num_threads):
worker = Thread(target=do_stuff, args=(q, ))
worker.setDaemon(True)
worker.start()
q.join()
print("time: ", time.time() - start) # ~20 seconds
The answer lies here:
for message_nbr in range(40):
q.put(f())
You're putting 40 instances of None into your queue, because you're calling f() which returns None instead of passing f (the function object) in. This block is what is taking 20 seconds to run!
Changing this code
def do_stuff(q):
while True:
output = q.get()
q.task_done()
to this
def do_stuff(q):
while True:
output = q.get()
output()
q.task_done()
is also necessary (you need to call the function!)
Final:
import time
from queue import Queue
from threading import Thread
start = time.time()
def f():
time.sleep(0.5)
print("yes")
return 'yes'
def do_stuff(q):
while True:
output = q.get()
output()
q.task_done()
q = Queue(maxsize=100)
for message_nbr in range(40):
q.put(f)
num_threads = 10
for i in range(num_threads):
worker = Thread(target=do_stuff, args=(q, ))
worker.setDaemon(True)
worker.start()
q.join()
print("time: ", time.time() - start) # time: 2.183439254760742
this one works !
start = time.time()
def f(m):
time.sleep(0.5)
print("yes")
return 'yes'
def do_stuff(q):
while True:
output = q.get()
final_result = f(output)
q.task_done()
q = Queue(maxsize=0)
for message_nbr in range(10):
# q.put(f())
q.put(message_nbr)
num_threads = 10
for i in range(num_threads):
worker = Thread(target=do_stuff, args=(q, ))
worker.setDaemon(True)
worker.start()
q.join()
print("time: ", time.time() - start)
Related
I have a program(python 3.9.10) that has a read queue and a write queue. One thread reads and once read, sends to the write queue and another thread writes.
All works fine unless there is an error. If there is, the threads do not stop.
In the following code I am simulating an error being detected in the read thread and trying to stop the threads from reading/writing so the program exits however the program/threads stay active and the program never finishes. If I remove the error simulation code, the threads stop and the program finishes.
I wish to handle the errors WITHIN the threads and if need be, stop the threads/program without throwing an error up
What am I doing wrong? Thanks
Here is a working example of my issue:
import pandas as pd
import datetime
import traceback
from queue import Queue
from threading import Thread
import time
dlQueue = Queue()
writeQueue = Queue()
dlQDone = False
errorStop = False
def log(text):
text = datetime.datetime.now().strftime("%Y/%m/%d, %H:%M:%S ") + text
print(text)
def errorBreak():
global dlQueue
global writeQueue
global errorStop
global dlQDone
dlQueue = Queue()
writeQueue = Queue()
errorStop = True
dlQDone = True
def downloadTable(t, q):
global dlQDone
global errorStop
while True:
if errorStop:
return
nextQ = q.get()
log("READING: " + nextQ)
writeQueue.put("Writing " + nextQ)
log("DONE READING: " + nextQ)
####sumulating an error and need to exit threads###
if nextQ == "Read 7":
log("Breaking Read")
errorBreak()
return
###################################################
q.task_done()
if q.qsize() == 0:
log("Download QUEUE finished")
dlQDone = True
return
def writeTable(t, q):
global errorStop
global dlQDone
while True:
if errorStop:
log("Error Stop return")
return
nextQ = q.get()
log("WRITING: " + nextQ)
log("DONE WRITING: " + nextQ)
q.task_done()
if dlQDone:
if q.qsize() == 0:
log("Writing QUEUE finished")
return
try:
log("PROCESS STARTING!!")
for i in range(10):
dlQueue.put("Read " + str(i))
startTime = time.time()
log("Starting threaded pull....")
dlWorker = Thread(
target=downloadTable,
args=(
"DL",
dlQueue,
),
)
dlWorker.start()
writeWorker = Thread(
target=writeTable,
args=(
"Write",
writeQueue,
),
)
writeWorker.start()
dlQueue.join()
writeQueue.join()
log(f"Finished thread in {str(time.time() - startTime)} seconds") # CANNOT GET HERE
log("Threads: " + str(dlWorker.is_alive()) + str(writeWorker.is_alive()))
except Exception as error:
log(error)
log(traceback.format_exc())
If I understood you correctly, you want to stop both threads in case there's some error that warrants it; you can do that with a threading.Event, and changing your queue reads to have a timeout.
import datetime
import time
import queue
import threading
dlQueue = queue.Queue()
writeQueue = queue.Queue()
stop_event = threading.Event()
def log(text):
text = datetime.datetime.now().strftime("%Y/%m/%d, %H:%M:%S ") + text
print(text)
def downloadTable(t: str, q: queue.Queue):
while not stop_event.is_set():
try:
nextQ = q.get(timeout=1)
except queue.Empty:
continue
log("READING: " + nextQ)
writeQueue.put("Writing " + nextQ)
log("DONE READING: " + nextQ)
if nextQ == "7":
log("Breaking Read")
stop_event.set()
break
q.task_done()
log("Download thread exiting")
def writeTable(t, q):
while not stop_event.is_set():
try:
nextQ = q.get(timeout=1)
except queue.Empty:
continue
log("WRITING: " + nextQ)
log("DONE WRITING: " + nextQ)
q.task_done()
log("Write thread exiting")
def main():
log("PROCESS STARTING!!")
for i in range(10):
dlQueue.put(f"{i}")
log("Starting threaded pull....")
dlWorker = threading.Thread(
target=downloadTable,
args=(
"DL",
dlQueue,
),
)
dlWorker.start()
writeWorker = threading.Thread(
target=writeTable,
args=(
"Write",
writeQueue,
),
)
writeWorker.start()
dlWorker.join()
writeWorker.join()
if __name__ == "__main__":
main()
I know there's something stupid I'm doing, but I can't figure this out! I'm testing by doing 1 thread. It's supposed to add up to ten items in the queue, wait for it to finish, and then say "Finished". The thread function has q.task_done() at the end of it. But it'll say "Task Done" but then it just pauses and doesn't ever say "Finished".
def helper(q):
print("Started")
....
q.task_done()
print("Task done")
def main():
q = Queue(maxsize=0)
recid = 9000000
num_threads = 1
for i in range(num_threads):
worker = Thread(target=helper, args=(q))
worker.setDaemon(True)
worker.start()
while recid > 0:
batch.clear()
for x in range(10):
q.put(recid)
print("Added request.")
recid -= 1
print("Waiting for threads to finish.")
q.join()
print("Finished")
worker was dead before you called q.join(). maybe try to add some delays in helper() waiting for q.put()
def helper(q):
print("Started")
time.sleep(1) # wait for main() q.put()
while not q.empty(): # until queue is empty
print("request", q.get())
q.task_done() # trigger q.join in main()
print("Task done")
time.sleep(1) # wait for next q.put() if exists
Updated:
Basically, q.task_done() in helper() triggers q.join() in main() and worker.join() will be triggered once helper() is exited. An example code is as followed:
def main():
q = Queue(maxsize=0)
recid = 100
num_threads = 4
while recid > 0:
for i in range(num_threads):
print('''******** Hired Worker %d ********'''%(i),)
worker = Thread(target=helper, args=(q,))
worker.setDaemon(True)
worker.start()
# batch.clear()
for x in range(3):
q.put(recid)
print("Request Worker %d to do %d."%(i, recid))
recid -= 1
##############################################################
# Comment these lines when you are ready to test multiThread
##############################################################
print("Waiting Worker %d to finish %d."%(i, recid))
q.join() # triggered by q.task_done in helper()
print("Worker %d Finished %d"%(i, recid))
print("Waiting for Worker %d to leave."%i)
worker.join() # triggered once helper() is exited
print("!!!!!!!! Worker %d left !!!!!!!!"%i)
##############################################################
time.sleep(5) # wait until all requests are finished.
print("Finished")
Once you are happy with join() of Thread and Queue. You may try multithreading by commenting that block of code.
I want to measure how much time it takes to finish running the code with multiple threads in python.
If I put join inside the loop, it will stop the loop (main thread) from keep creating new threads. It will run the sleep() one by one.
If I put join on the thread which I use to create thread_testing, the join won't work somehow. It prints out the time immediately.
def sleep(name):
print("{} going to sleep".format(name))
time.sleep(5)
print("{} wakes up after 5 seconds".format(name))
def thread_testing():
for i in range(3):
t = threading.Thread(target=sleep, name='thread' + str(i), args=(i,)
t.start()
# t.join() #1
if __name__ == '__main__':
start = time.time()
t = threading.Thread(target=thread_testing, name='threadx')
t.start()
t.join() #2
print(time.time() - start)
Desired output:
1 sleep
2 sleep
3 sleep
1 wake up after 5
2 wake up after 5
3 wake up after 5
5.xxx secs
Join will wait for your thread. That is why your threads were executed one by one.
What you have to do is:
Start all threads
Store them somewhere
Once everything is started wait for every thread to finish.
Assuming you don't need the first thread started in main:
import time
import threading
def sleep(name):
print("{} going to sleep".format(name))
time.sleep(5)
print("{} wakes up after 5 seconds".format(name))
def thread_testing():
threads = []
for i in range(3):
t = threading.Thread(target=sleep, name='thread' + str(i), args=(i,))
t.start()
threads.append(t)
for t in threads:
t.join()
if __name__ == '__main__':
start = time.time()
thread_testing()
print(time.time() - start)
Basically, I have a async worker doing stuff, but I can caught an user command line any time.
The problem is just visual, when the user is typing and the code prints something, the text the user was typing is lifted up together with the printed text. How to make the last line "isolated" from console?
this is an exemple code:
import queue
import threading
import time
import heapq
def worker():
while True:
item = q.get()
if item is None:
break
do_work(item)
#q.task_done()
time.sleep(2)
q.put(item)
def do_work(item):
print(item,end = '')
print(time.time(), end = '')
q = queue.PriorityQueue()
num_worker_threads = 1
threads = []
for i in range(num_worker_threads):
t = threading.Thread(target=worker)
t.start()
threads.append(t)
fruits = [(1,"apple"), (2,"banana"), (3,"cherry")]
for x in fruits:
q.put(x)
gameFinished = 0
# block until all tasks are done
#q.join()
while not gameFinished:
q.put((-1,input()))
# stop workers
for i in range(num_worker_threads):
q.put(None)
for t in threads:
t.join()
input("Press enter to exit ;)")
I am new with python and I am trying to made code that stop work in case timeout has been reached. But it seems that the tread correctly timeout but it does not stop the work.
Here is my code:
import threading
import time
import sys
def main():
t1_stop= threading.Event()
t1 = threading.Thread(target=thread1, args=(5, t1_stop))
t1.setDaemon(False)
t1.start()
print 'thread 1 set'
while True:
print "blablabla"
time.sleep(1)
def thread1(time, stop_event):
while(not stop_event.is_set()):
#equivalent to time.sleep()
print 'thread 1'
stop_event.wait(time)
main()
UPD
I update code using Timer instead of time.time.
def main():
stopped = threading.Event()
timeout = 10
#thread = threading.Thread(target=my_thread, args=(timeout, stopped))
timer = Timer(timeout, my_thread(timeout,stopped))
thread = threading.Thread((timer).start())
thread.setDaemon(False)
#thread.start()
print 'thread 1 set'
start_t = time.time()
while thread.is_alive():
print "doing my job"
if not stopped.is_set():# and (time.time() - start_t) > timeout:
stopped.set()
#time.sleep(1)
def my_thread(time, stopped):
while not stopped.wait(time):
print('thread stopped')
main()
But I still get original problem the script does not stopped and continue.
Thanks in advance for your help.
you must call t1_stop.set() in main function for stop thread.
something like:
import threading
import time
import sys
def main():
stopped = threading.Event()
thread = threading.Thread(target=my_thread, args=(5, stopped))
thread.setDaemon(False)
thread.start()
print 'thread 1 set'
time.sleep(5) # +
stopped.set() # +
while True:
print "blablabla"
time.sleep(1)
def my_thread(time, stopped):
while not stopped.wait(time):
print('thread 1')
main()
with 'blablabla':
def main():
...
thread.start()
print 'thread 1 set'
start_t = time.time()
while True:
print "blablabla"
if not stopped.is_set() and (time.time() - start_t) > 5:
stopped.set()
time.sleep(1)
UPD:
exiting from while:
while thread.is_alive():
print "blablabla"
if not stopped.is_set() and (time.time() - start_t) > 5:
stopped.set()
time.sleep(1)