I have a python script that uses threading, here is my code:
def main():
ip_list = get_ip_list() #['ip1', 'ip2'..., 'ipn'] thousands
for x in ip_list:
q.put(x)
threads = []
threads_num = 50
for x in range(threads_num):
w = Worker(q, stantard_config_to_dict, id_dict, logger,ip_position)
threads.append(w)
w.start()
for w in threads:
w.join()
logger.debug('End of main thread, exit')
print 'End of main thread, exit'
def run(self):
while 1:
try:
ip = self.queue.get(False)
self.logger.debug('%s get %s from queue, left %s in queue' % (self.getName(), ip, self.queue.qsize()))
self.get_inter_conf(ip)
except Queue.Empty:
self.logger.debug('queue is empty, exit')
break
self.logger.info("%s finished" % (self.getName()))
The string 'End of main thread' doesn't get printed right now, only when I change the number of items in ip_list it gets printed.
Why does this happen?
Related
I have a program(python 3.9.10) that has a read queue and a write queue. One thread reads and once read, sends to the write queue and another thread writes.
All works fine unless there is an error. If there is, the threads do not stop.
In the following code I am simulating an error being detected in the read thread and trying to stop the threads from reading/writing so the program exits however the program/threads stay active and the program never finishes. If I remove the error simulation code, the threads stop and the program finishes.
I wish to handle the errors WITHIN the threads and if need be, stop the threads/program without throwing an error up
What am I doing wrong? Thanks
Here is a working example of my issue:
import pandas as pd
import datetime
import traceback
from queue import Queue
from threading import Thread
import time
dlQueue = Queue()
writeQueue = Queue()
dlQDone = False
errorStop = False
def log(text):
text = datetime.datetime.now().strftime("%Y/%m/%d, %H:%M:%S ") + text
print(text)
def errorBreak():
global dlQueue
global writeQueue
global errorStop
global dlQDone
dlQueue = Queue()
writeQueue = Queue()
errorStop = True
dlQDone = True
def downloadTable(t, q):
global dlQDone
global errorStop
while True:
if errorStop:
return
nextQ = q.get()
log("READING: " + nextQ)
writeQueue.put("Writing " + nextQ)
log("DONE READING: " + nextQ)
####sumulating an error and need to exit threads###
if nextQ == "Read 7":
log("Breaking Read")
errorBreak()
return
###################################################
q.task_done()
if q.qsize() == 0:
log("Download QUEUE finished")
dlQDone = True
return
def writeTable(t, q):
global errorStop
global dlQDone
while True:
if errorStop:
log("Error Stop return")
return
nextQ = q.get()
log("WRITING: " + nextQ)
log("DONE WRITING: " + nextQ)
q.task_done()
if dlQDone:
if q.qsize() == 0:
log("Writing QUEUE finished")
return
try:
log("PROCESS STARTING!!")
for i in range(10):
dlQueue.put("Read " + str(i))
startTime = time.time()
log("Starting threaded pull....")
dlWorker = Thread(
target=downloadTable,
args=(
"DL",
dlQueue,
),
)
dlWorker.start()
writeWorker = Thread(
target=writeTable,
args=(
"Write",
writeQueue,
),
)
writeWorker.start()
dlQueue.join()
writeQueue.join()
log(f"Finished thread in {str(time.time() - startTime)} seconds") # CANNOT GET HERE
log("Threads: " + str(dlWorker.is_alive()) + str(writeWorker.is_alive()))
except Exception as error:
log(error)
log(traceback.format_exc())
If I understood you correctly, you want to stop both threads in case there's some error that warrants it; you can do that with a threading.Event, and changing your queue reads to have a timeout.
import datetime
import time
import queue
import threading
dlQueue = queue.Queue()
writeQueue = queue.Queue()
stop_event = threading.Event()
def log(text):
text = datetime.datetime.now().strftime("%Y/%m/%d, %H:%M:%S ") + text
print(text)
def downloadTable(t: str, q: queue.Queue):
while not stop_event.is_set():
try:
nextQ = q.get(timeout=1)
except queue.Empty:
continue
log("READING: " + nextQ)
writeQueue.put("Writing " + nextQ)
log("DONE READING: " + nextQ)
if nextQ == "7":
log("Breaking Read")
stop_event.set()
break
q.task_done()
log("Download thread exiting")
def writeTable(t, q):
while not stop_event.is_set():
try:
nextQ = q.get(timeout=1)
except queue.Empty:
continue
log("WRITING: " + nextQ)
log("DONE WRITING: " + nextQ)
q.task_done()
log("Write thread exiting")
def main():
log("PROCESS STARTING!!")
for i in range(10):
dlQueue.put(f"{i}")
log("Starting threaded pull....")
dlWorker = threading.Thread(
target=downloadTable,
args=(
"DL",
dlQueue,
),
)
dlWorker.start()
writeWorker = threading.Thread(
target=writeTable,
args=(
"Write",
writeQueue,
),
)
writeWorker.start()
dlWorker.join()
writeWorker.join()
if __name__ == "__main__":
main()
I want to stop asynchronous multiprocessing jobs with KeyboardInterrupt. But sometimes hang occurred when call terminate.
from multiprocessing.pool import ThreadPool
import multiprocessing
import time
import queue
import inspect
def worker(index):
print('{}: start'.format(index))
for i in range(5):
time.sleep(1)
print('{}: stop'.format(index))
return index, True
def wrapper(index, stopEvent, qResult):
if stopEvent.is_set() is True:
return index, False
try:
result = worker(index)
except:
print('*' * 50)
return index, False
else:
if result[1] == True:
qResult.put(result)
return result
def watcher(qResult, stopEvent):
cntQ = 0
while True:
try:
result = qResult.get(timeout=10)
qResult.task_done()
except queue.Empty:
if stopEvent.is_set() is True:
break
except KeyboardInterrupt:
stopEvent.set()
else:
cntQ += 1
print(result)
qResult.join()
qResult.close()
print('qResult count:', cntQ)
def main():
stopEvent = multiprocessing.Event()
qResult = multiprocessing.JoinableQueue()
qResult.cancel_join_thread()
watch = multiprocessing.Process(target=watcher, args=(qResult, stopEvent))
watch.start()
pool = ThreadPool()
lsRet = []
for i in range(100000):
try:
ret = pool.apply_async(wrapper, args=(i, stopEvent, qResult))
lsRet.append(ret)
except KeyboardInterrupt:
stopEvent.set()
time.sleep(1)
break
if i+1 % 10 == 0:
time.sleep(2)
cntTotal = len(lsRet)
cntRet = 0
for ret in lsRet:
if stopEvent.is_set():
break
try:
ret.get()
except KeyboardInterrupt:
stopEvent.set()
time.sleep(1)
else:
cntRet += 1
if stopEvent.is_set() is False:
stopEvent.set()
print(inspect.stack()[0][1:4])
if watch.is_alive() is True:
watch.join()
print(inspect.stack()[0][1:4])
pool.terminate() # Why hang??????????
print(inspect.stack()[0][1:4])
pool.join()
print(cntTotal, cntRet)
if __name__ == '__main__':
main()
main() invokes a watcher() thread and many wrapper() threads asynchronously using multiprocessing.pool.Threadpool.
wrapper() calls worker() and put its result to queue.
watcher() watches above queue of results.
If ctrl-c pressed, stopEvent is set.
When stopEvent is set, wrapper() stops calling worker(), and Watcher() indicates queue.Empty and stopEvent and exits loop.
Finally main() calls terminate() of pool.
Sometimes processes done well, but sometimes hang. It's different each time.
You should put the code in try except block and catch a built-in exception KeyboardInterrupt see the example here Capture keyboardinterrupt
Thanks to #user5402 for the previous solution.
I am trying to handle multiple messages that are queued up. Here is the code:
import sys
import socket
from multiprocessing import Process, Queue
UDP_ADDR = ("", 13000)
def send(m):
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.sendto(m, UDP_ADDR)
def receive(q):
buf = 1024
Sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
Sock.bind(UDP_ADDR)
while True:
(data, addr) = Sock.recvfrom(buf)
q.put(data)
In the client function I want to handle multiple messages, that have knock on affects.
def client():
q = Queue()
r = Process(target = receive, args=(q,))
r.start()
print "client loop started"
while True:
m = q.get()
print "got:", m
while m == "start":
print "started"
z = q.get()
if z == "stop":
return
print "loop ended"
r.terminate()
So when start is sent, it then goes into a while loop that is infinitely printing "started", and waiting for the stop message to come through. The above client code does not work.
Here is the code to start the function:
if __name__ == '__main__':
args = sys.argv
if len(args) > 1:
send(args[1])
else:
client()
You can write the client loop like this:
print "client loop started"
while True:
m = q.get()
print "waiting for start, got:", m
if m == "start":
while True:
try:
m = q.get(False)
except:
m = None
print "waiting for stop, got:", m
if m == "stop":
break
Based on your comments, this would be a better approach:
import sys
import socket
import Queue as Q
import time
from multiprocessing import Process, Queue
UDP_ADDR = ("", 13000)
def send(m):
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.sendto(m, UDP_ADDR)
def receive(q):
buf = 1024
Sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
Sock.bind(UDP_ADDR)
while True:
(data, addr) = Sock.recvfrom(buf)
q.put(data)
def doit():
# ... what the processing thread will do ...
while True:
print "sleeping..."
time.sleep(3)
def client():
q = Queue()
r = Process(target = receive, args=(q,))
r.start()
print "client loop started"
t = None # the processing thread
while True:
m = q.get()
if m == "start":
if t:
print "processing thread already started"
else:
t = Process(target = doit)
t.start()
print "processing thread started"
elif m == "stop":
if t:
t.terminate()
t = None
print "processing thread stopped"
else:
print "processing thread not running"
elif m == "quit":
print "shutting down"
if t:
t.terminate()
t = None # play it safe
break
else:
print "huh?"
r.terminate()
if __name__ == '__main__':
args = sys.argv
if len(args) > 1:
send(args[1])
else:
client()
This code is to start and stop multiple copies of various different thread types. I will preface this with saying I tried using pipes to control the threads but keep getting random memory errors to do with the pipes. This is a prototype of a factory to produce several copies of multiple thread types, so a queue method didn't seem practical to me either though if there is something I am missing I am all ears. So my newest plan is to use a global dictionary with a different entry for every thread and thread kill condition. ie sub[Alive] subKill["kill"]
Yet for some reason the control_listener thread spawned by the processes wont trigger the kill condition and wont read the global variable.
from multiprocessing import Process, Pipe
from threading import Thread
import time
alive = {'subAlive': True, 'subKill': "Alive", 'testAlive': True, 'testKill': "Alive"};
def control_listener(aliveFlag, threadAlive): #listens for kill from main
global alive
while True:
data = alive[aliveFlag];
print "Thread", alive[threadAlive];
print "Thread status", alive[aliveFlag];
if data == "kill":
print "Killing"
alive[threadAlive] = False; #value for kill
print "testListner alive %s" % threadAlive, alive[threadAlive];
print "deactivating %s" % threadAlive, alive['aliveFlag'];
break
def subprocess(aliveFlag, threadNum, threadAlive):
t = Thread(target=control_listener, args=(aliveFlag, threadAlive))
count = 0
threadVal = threadNum
t.start()
run = alive['subAlive'];
while run == True:
print "Thread alive %s" % alive['aliveFlag'];
print "Thread %d Run number = %d" % (threadVal, count), alive['subAlive'];
count = count + 1
run = alive['subAlive'];
def testprocess(aliveFlag, threadNum, threadAlive):
t = Thread(target=control_listener, args=(aliveFlag, threadAlive))
count = 0
threadVal = threadNum
t.start()
run = alive['testAlive'];
while run == True:
print "This is a different thread %d Run = %d" % (threadVal, count)
count = count + 1
run = alive['testAlive'];
runNum = int(raw_input("Enter a number: "))
threadNum = int(raw_input("Enter number of threads: "))
print "Starting threads"
for i in range(threadNum):
p = Process(target=subprocess, args=('subKill', i, 'subAlive'))
p.start()
print "Subprocess started"
for i in range(threadNum):
p2 = Process(target=testprocess, args=('subKill', i, 'testAlive'))
p2.start()
print "Testproccess started"
print "Starting run"
time.sleep(runNum)
print "Terminating Subprocess run"
for i in range(threadNum):
alive['subKill'] = "kill";
print "Subkill = %s" % alive['subKill'];
print "Testprocess termination alive", alive['subAlive'];
print "Terminating Testprocess run"
for i in range(threadNum):
alive['subKill'] = "kill";
print "Testprocess termination alive", alive['subAlive'];
p.join()
p2.join()
I have this script and it does work it has 2 separate processes that spawn listener threads to kill the process when kill is sent to the listener via a pipe.
from multiprocessing import Process, Pipe
from threading import Thread
import time
subAlive = True
testAlive = True
def sub_listener(conn): #listens for kill from main
global subAlive
while True:
data = conn.recv()
if data == "kill":
subAlive = False #value for kill
break
def test_listener(conn): #listens for kill from main
global testAlive
while True:
data = conn.recv()
if data == "kill":
testAlive = False #value for kill
break
def subprocess(conn, threadNum):
t = Thread(target=sub_listener, args=(conn,))
count = 0
threadVal = threadNum
t.start()
while subAlive:
print "Thread %d Run number = %d" % (threadVal, count)
count = count + 1
def testprocess(conn, threadNum):
t = Thread(target=test_listener, args=(conn,))
count = 0
threadVal = threadNum
t.start()
while testAlive:
print "This is a different thread %d Run = %d" % (threadVal, count)
count = count + 1
sub_parent, sub_child = Pipe()
test_parent, test_child = Pipe()
runNum = int(raw_input("Enter a number: "))
threadNum = int(raw_input("Enter number of threads: "))
print "Starting threads"
for i in range(threadNum):
p = Process(target=subprocess, args=(sub_child, i))
p.start()
print "Subprocess started"
for i in range(threadNum):
p2 = Process(target=testprocess, args=(test_child, i))
p2.start()
print "Testproccess started"
print "Starting run"
time.sleep(runNum)
print "Terminating Subprocess run"
for i in range(threadNum):
sub_parent.send("kill") #sends kill to listener
print "Terminating Testprocess run"
for i in range(threadNum):
test_parent.send("kill") #sends kill to listener
p.join()
p2.join()
Id like to not need a separate listener function hard coded for every process I call. I was thinking about passing global variables when the thread is spawned. The global variables are really the only differences between the listener functions. Thanks guys!
You can access globals through the globals() dictionary.
>>> foo = 'value'
>>> def change(name):
... globals()[name] = 'changed'
...
>>> change('foo')
>>> foo
'changed'
but I would suggest:
alive = {}
def sub_listener(conn, key): #listens for kill from main
while True:
data = conn.recv()
if data == "kill":
alive[key] = False #value for kill
break
e.g.
from multiprocessing import Process, Pipe
from threading import Thread
import time
alive = {
'sub': 1,
'test': 1,
}
def listener_factory(key):
def listener(conn): #listens for kill from main
while True:
data = conn.recv()
if data == "kill":
alive[key] = False #value for kill
break
return listener
def process_factory(key):
listener = listener_factory(key)
def subprocess(conn, threadNum):
t = Thread(target=listener, args=(conn,))
count = 0
threadVal = threadNum
t.start()
while alive[key]:
print "Thread[%s] %d Run number = %d" % (key, threadVal, count)
count = count + 1
return subprocess
def main():
sub_parent, sub_child = Pipe()
test_parent, test_child = Pipe()
runNum = int(raw_input("Enter a number: "))
threadNum = int(raw_input("Enter number of threads: "))
print "Starting threads"
for i in range(threadNum):
p = Process(target=process_factory('sub'), args=(sub_child, i))
p.start()
print "Subprocess started"
for i in range(threadNum):
p2 = Process(target=process_factory('test'), args=(test_child, i))
p2.start()
print "Testproccess started"
print "Starting run"
time.sleep(runNum)
print "Terminating Subprocess run"
for i in range(threadNum):
sub_parent.send("kill") #sends kill to listener
print "Terminating Testprocess run"
for i in range(threadNum):
test_parent.send("kill") #sends kill to listener
p.join()
p2.join()
if __name__ == '__main__':
main()