Develop Reference

How to stop threads using Queue()

I have a program(python 3.9.10) that has a read queue and a write queue. One thread reads and once read, sends to the write queue and another thread writes.
All works fine unless there is an error. If there is, the threads do not stop.
In the following code I am simulating an error being detected in the read thread and trying to stop the threads from reading/writing so the program exits however the program/threads stay active and the program never finishes. If I remove the error simulation code, the threads stop and the program finishes.
I wish to handle the errors WITHIN the threads and if need be, stop the threads/program without throwing an error up
What am I doing wrong? Thanks
Here is a working example of my issue:
import pandas as pd
import datetime
import traceback
from queue import Queue
from threading import Thread
import time
dlQueue = Queue()
writeQueue = Queue()
dlQDone = False
errorStop = False
def log(text):
text = datetime.datetime.now().strftime("%Y/%m/%d, %H:%M:%S ") + text
print(text)
def errorBreak():
global dlQueue
global writeQueue
global errorStop
global dlQDone
dlQueue = Queue()
writeQueue = Queue()
errorStop = True
dlQDone = True
def downloadTable(t, q):
global dlQDone
global errorStop
while True:
if errorStop:
return
nextQ = q.get()
log("READING: " + nextQ)
writeQueue.put("Writing " + nextQ)
log("DONE READING: " + nextQ)
####sumulating an error and need to exit threads###
if nextQ == "Read 7":
log("Breaking Read")
errorBreak()
return
###################################################
q.task_done()
if q.qsize() == 0:
log("Download QUEUE finished")
dlQDone = True
return
def writeTable(t, q):
global errorStop
global dlQDone
while True:
if errorStop:
log("Error Stop return")
return
nextQ = q.get()
log("WRITING: " + nextQ)
log("DONE WRITING: " + nextQ)
q.task_done()
if dlQDone:
if q.qsize() == 0:
log("Writing QUEUE finished")
return
try:
log("PROCESS STARTING!!")
for i in range(10):
dlQueue.put("Read " + str(i))
startTime = time.time()
log("Starting threaded pull....")
dlWorker = Thread(
target=downloadTable,
args=(
"DL",
dlQueue,
),
)
dlWorker.start()
writeWorker = Thread(
target=writeTable,
args=(
"Write",
writeQueue,
),
)
writeWorker.start()
dlQueue.join()
writeQueue.join()
log(f"Finished thread in {str(time.time() - startTime)} seconds") # CANNOT GET HERE
log("Threads: " + str(dlWorker.is_alive()) + str(writeWorker.is_alive()))
except Exception as error:
log(error)
log(traceback.format_exc())

If I understood you correctly, you want to stop both threads in case there's some error that warrants it; you can do that with a threading.Event, and changing your queue reads to have a timeout.
import datetime
import time
import queue
import threading
dlQueue = queue.Queue()
writeQueue = queue.Queue()
stop_event = threading.Event()
def log(text):
text = datetime.datetime.now().strftime("%Y/%m/%d, %H:%M:%S ") + text
print(text)
def downloadTable(t: str, q: queue.Queue):
while not stop_event.is_set():
try:
nextQ = q.get(timeout=1)
except queue.Empty:
continue
log("READING: " + nextQ)
writeQueue.put("Writing " + nextQ)
log("DONE READING: " + nextQ)
if nextQ == "7":
log("Breaking Read")
stop_event.set()
break
q.task_done()
log("Download thread exiting")
def writeTable(t, q):
while not stop_event.is_set():
try:
nextQ = q.get(timeout=1)
except queue.Empty:
continue
log("WRITING: " + nextQ)
log("DONE WRITING: " + nextQ)
q.task_done()
log("Write thread exiting")
def main():
log("PROCESS STARTING!!")
for i in range(10):
dlQueue.put(f"{i}")
log("Starting threaded pull....")
dlWorker = threading.Thread(
target=downloadTable,
args=(
"DL",
dlQueue,
),
)
dlWorker.start()
writeWorker = threading.Thread(
target=writeTable,
args=(
"Write",
writeQueue,
),
)
writeWorker.start()
dlWorker.join()
writeWorker.join()
if __name__ == "__main__":
main()

python threading not exit

I have a python script that uses threading, here is my code:
def main():
ip_list = get_ip_list() #['ip1', 'ip2'..., 'ipn'] thousands
for x in ip_list:
q.put(x)
threads = []
threads_num = 50
for x in range(threads_num):
w = Worker(q, stantard_config_to_dict, id_dict, logger,ip_position)
threads.append(w)
w.start()
for w in threads:
w.join()
logger.debug('End of main thread, exit')
print 'End of main thread, exit'
def run(self):
while 1:
try:
ip = self.queue.get(False)
self.logger.debug('%s get %s from queue, left %s in queue' % (self.getName(), ip, self.queue.qsize()))
self.get_inter_conf(ip)
except Queue.Empty:
self.logger.debug('queue is empty, exit')
break
self.logger.info("%s finished" % (self.getName()))
The string 'End of main thread' doesn't get printed right now, only when I change the number of items in ip_list it gets printed.
Why does this happen?

Python thread not starting.

I am having issues with my halt_listener thread. I can start import_1 but it will not spawn a halt_listener thread. I am patterning this after known good code, the only difference was in the last iteration the halt_listener got fed a pipe instead of a queue.
class test_imports:#Test classes remove
alive = {'import_1': True, 'import_2': True};
def halt_listener(self, control_Queue, thread_Name, kill_command):
while True:
print ("Checking queue for kill")
isAlive = control_queue.get()
print ("isAlive", isAlive)
if isAlive == kill_command:
print ("kill listener triggered")
self.alive[thread_Name] = False;
return
def import_1(self, control_Queue, thread_Number):
print ("Import_1 number %d started") % thread_Number
t = Thread(target=test_imports.halt_listener, args=(control_Queue, 'import_1', 't1kill'))
count = 0
global alive
run = test_imports.alive['import_1'];
while run:
print ("Thread type 1 number %d run count %d") % (thread_Number, count)
count = count + 1
print ("Test Import_1 ", run)
run = self.alive['import_1'];
print ("Killing thread type 1 number %d") % thread_Number
def import_2(self, control_queue, thread_number):
print ("Import_2 number %d started") % thread_number
count = 1
while True:
alive = control_queue.get()
count = count + 1
if alive == 't2kill':
print ("Killing thread type 2 number %d") % thread_number
return
else:
print ("Thread type 2 number %d run count %d") % (thread_number, count)
Anyone have any pointers to where I am going wrong.

You never call t.start() to actually start the thread.

everything is okay, you just have to add t.start() to start your thread
t = Thread(target=test_imports.halt_listener, args=(control_Queue, 'import_1', 't1kill'))
t.start()

Why wont my python thread react to global variables

This code is to start and stop multiple copies of various different thread types. I will preface this with saying I tried using pipes to control the threads but keep getting random memory errors to do with the pipes. This is a prototype of a factory to produce several copies of multiple thread types, so a queue method didn't seem practical to me either though if there is something I am missing I am all ears. So my newest plan is to use a global dictionary with a different entry for every thread and thread kill condition. ie sub[Alive] subKill["kill"]
Yet for some reason the control_listener thread spawned by the processes wont trigger the kill condition and wont read the global variable.
from multiprocessing import Process, Pipe
from threading import Thread
import time
alive = {'subAlive': True, 'subKill': "Alive", 'testAlive': True, 'testKill': "Alive"};
def control_listener(aliveFlag, threadAlive): #listens for kill from main
global alive
while True:
data = alive[aliveFlag];
print "Thread", alive[threadAlive];
print "Thread status", alive[aliveFlag];
if data == "kill":
print "Killing"
alive[threadAlive] = False; #value for kill
print "testListner alive %s" % threadAlive, alive[threadAlive];
print "deactivating %s" % threadAlive, alive['aliveFlag'];
break
def subprocess(aliveFlag, threadNum, threadAlive):
t = Thread(target=control_listener, args=(aliveFlag, threadAlive))
count = 0
threadVal = threadNum
t.start()
run = alive['subAlive'];
while run == True:
print "Thread alive %s" % alive['aliveFlag'];
print "Thread %d Run number = %d" % (threadVal, count), alive['subAlive'];
count = count + 1
run = alive['subAlive'];
def testprocess(aliveFlag, threadNum, threadAlive):
t = Thread(target=control_listener, args=(aliveFlag, threadAlive))
count = 0
threadVal = threadNum
t.start()
run = alive['testAlive'];
while run == True:
print "This is a different thread %d Run = %d" % (threadVal, count)
count = count + 1
run = alive['testAlive'];
runNum = int(raw_input("Enter a number: "))
threadNum = int(raw_input("Enter number of threads: "))
print "Starting threads"
for i in range(threadNum):
p = Process(target=subprocess, args=('subKill', i, 'subAlive'))
p.start()
print "Subprocess started"
for i in range(threadNum):
p2 = Process(target=testprocess, args=('subKill', i, 'testAlive'))
p2.start()
print "Testproccess started"
print "Starting run"
time.sleep(runNum)
print "Terminating Subprocess run"
for i in range(threadNum):
alive['subKill'] = "kill";
print "Subkill = %s" % alive['subKill'];
print "Testprocess termination alive", alive['subAlive'];
print "Terminating Testprocess run"
for i in range(threadNum):
alive['subKill'] = "kill";
print "Testprocess termination alive", alive['subAlive'];
p.join()
p2.join()

How to streamline this script

I have this script and it does work it has 2 separate processes that spawn listener threads to kill the process when kill is sent to the listener via a pipe.
from multiprocessing import Process, Pipe
from threading import Thread
import time
subAlive = True
testAlive = True
def sub_listener(conn): #listens for kill from main
global subAlive
while True:
data = conn.recv()
if data == "kill":
subAlive = False #value for kill
break
def test_listener(conn): #listens for kill from main
global testAlive
while True:
data = conn.recv()
if data == "kill":
testAlive = False #value for kill
break
def subprocess(conn, threadNum):
t = Thread(target=sub_listener, args=(conn,))
count = 0
threadVal = threadNum
t.start()
while subAlive:
print "Thread %d Run number = %d" % (threadVal, count)
count = count + 1
def testprocess(conn, threadNum):
t = Thread(target=test_listener, args=(conn,))
count = 0
threadVal = threadNum
t.start()
while testAlive:
print "This is a different thread %d Run = %d" % (threadVal, count)
count = count + 1
sub_parent, sub_child = Pipe()
test_parent, test_child = Pipe()
runNum = int(raw_input("Enter a number: "))
threadNum = int(raw_input("Enter number of threads: "))
print "Starting threads"
for i in range(threadNum):
p = Process(target=subprocess, args=(sub_child, i))
p.start()
print "Subprocess started"
for i in range(threadNum):
p2 = Process(target=testprocess, args=(test_child, i))
p2.start()
print "Testproccess started"
print "Starting run"
time.sleep(runNum)
print "Terminating Subprocess run"
for i in range(threadNum):
sub_parent.send("kill") #sends kill to listener
print "Terminating Testprocess run"
for i in range(threadNum):
test_parent.send("kill") #sends kill to listener
p.join()
p2.join()
Id like to not need a separate listener function hard coded for every process I call. I was thinking about passing global variables when the thread is spawned. The global variables are really the only differences between the listener functions. Thanks guys!

You can access globals through the globals() dictionary.
>>> foo = 'value'
>>> def change(name):
... globals()[name] = 'changed'
...
>>> change('foo')
>>> foo
'changed'
but I would suggest:
alive = {}
def sub_listener(conn, key): #listens for kill from main
while True:
data = conn.recv()
if data == "kill":
alive[key] = False #value for kill
break
e.g.
from multiprocessing import Process, Pipe
from threading import Thread
import time
alive = {
'sub': 1,
'test': 1,
}
def listener_factory(key):
def listener(conn): #listens for kill from main
while True:
data = conn.recv()
if data == "kill":
alive[key] = False #value for kill
break
return listener
def process_factory(key):
listener = listener_factory(key)
def subprocess(conn, threadNum):
t = Thread(target=listener, args=(conn,))
count = 0
threadVal = threadNum
t.start()
while alive[key]:
print "Thread[%s] %d Run number = %d" % (key, threadVal, count)
count = count + 1
return subprocess
def main():
sub_parent, sub_child = Pipe()
test_parent, test_child = Pipe()
runNum = int(raw_input("Enter a number: "))
threadNum = int(raw_input("Enter number of threads: "))
print "Starting threads"
for i in range(threadNum):
p = Process(target=process_factory('sub'), args=(sub_child, i))
p.start()
print "Subprocess started"
for i in range(threadNum):
p2 = Process(target=process_factory('test'), args=(test_child, i))
p2.start()
print "Testproccess started"
print "Starting run"
time.sleep(runNum)
print "Terminating Subprocess run"
for i in range(threadNum):
sub_parent.send("kill") #sends kill to listener
print "Terminating Testprocess run"
for i in range(threadNum):
test_parent.send("kill") #sends kill to listener
p.join()
p2.join()
if __name__ == '__main__':
main()