Develop Reference

python multithreading with queue running in sequence not in parallel

why this code is not running in parallel, it take 20 seconds to run it, which means it is running in sequence. Thank you for your help.
import time
from queue import Queue
from threading import Thread
start = time.time()
def f():
time.sleep(0.5)
print("yes")
return 'yes'
def do_stuff(q):
while True:
output = q.get()
q.task_done()
q = Queue(maxsize=100)
for message_nbr in range(40):
q.put(f())
num_threads = 10
for i in range(num_threads):
worker = Thread(target=do_stuff, args=(q, ))
worker.setDaemon(True)
worker.start()
q.join()
print("time: ", time.time() - start) # ~20 seconds

The answer lies here:
for message_nbr in range(40):
q.put(f())
You're putting 40 instances of None into your queue, because you're calling f() which returns None instead of passing f (the function object) in. This block is what is taking 20 seconds to run!
Changing this code
def do_stuff(q):
while True:
output = q.get()
q.task_done()
to this
def do_stuff(q):
while True:
output = q.get()
output()
q.task_done()
is also necessary (you need to call the function!)
Final:
import time
from queue import Queue
from threading import Thread
start = time.time()
def f():
time.sleep(0.5)
print("yes")
return 'yes'
def do_stuff(q):
while True:
output = q.get()
output()
q.task_done()
q = Queue(maxsize=100)
for message_nbr in range(40):
q.put(f)
num_threads = 10
for i in range(num_threads):
worker = Thread(target=do_stuff, args=(q, ))
worker.setDaemon(True)
worker.start()
q.join()
print("time: ", time.time() - start) # time: 2.183439254760742

this one works !
start = time.time()
def f(m):
time.sleep(0.5)
print("yes")
return 'yes'
def do_stuff(q):
while True:
output = q.get()
final_result = f(output)
q.task_done()
q = Queue(maxsize=0)
for message_nbr in range(10):
# q.put(f())
q.put(message_nbr)
num_threads = 10
for i in range(num_threads):
worker = Thread(target=do_stuff, args=(q, ))
worker.setDaemon(True)
worker.start()
q.join()
print("time: ", time.time() - start)

Why does terminate() of multiprocessing.pool.Threadpool hang?

I want to stop asynchronous multiprocessing jobs with KeyboardInterrupt. But sometimes hang occurred when call terminate.
from multiprocessing.pool import ThreadPool
import multiprocessing
import time
import queue
import inspect
def worker(index):
print('{}: start'.format(index))
for i in range(5):
time.sleep(1)
print('{}: stop'.format(index))
return index, True
def wrapper(index, stopEvent, qResult):
if stopEvent.is_set() is True:
return index, False
try:
result = worker(index)
except:
print('*' * 50)
return index, False
else:
if result[1] == True:
qResult.put(result)
return result
def watcher(qResult, stopEvent):
cntQ = 0
while True:
try:
result = qResult.get(timeout=10)
qResult.task_done()
except queue.Empty:
if stopEvent.is_set() is True:
break
except KeyboardInterrupt:
stopEvent.set()
else:
cntQ += 1
print(result)
qResult.join()
qResult.close()
print('qResult count:', cntQ)
def main():
stopEvent = multiprocessing.Event()
qResult = multiprocessing.JoinableQueue()
qResult.cancel_join_thread()
watch = multiprocessing.Process(target=watcher, args=(qResult, stopEvent))
watch.start()
pool = ThreadPool()
lsRet = []
for i in range(100000):
try:
ret = pool.apply_async(wrapper, args=(i, stopEvent, qResult))
lsRet.append(ret)
except KeyboardInterrupt:
stopEvent.set()
time.sleep(1)
break
if i+1 % 10 == 0:
time.sleep(2)
cntTotal = len(lsRet)
cntRet = 0
for ret in lsRet:
if stopEvent.is_set():
break
try:
ret.get()
except KeyboardInterrupt:
stopEvent.set()
time.sleep(1)
else:
cntRet += 1
if stopEvent.is_set() is False:
stopEvent.set()
print(inspect.stack()[0][1:4])
if watch.is_alive() is True:
watch.join()
print(inspect.stack()[0][1:4])
pool.terminate() # Why hang??????????
print(inspect.stack()[0][1:4])
pool.join()
print(cntTotal, cntRet)
if __name__ == '__main__':
main()
main() invokes a watcher() thread and many wrapper() threads asynchronously using multiprocessing.pool.Threadpool.
wrapper() calls worker() and put its result to queue.
watcher() watches above queue of results.
If ctrl-c pressed, stopEvent is set.
When stopEvent is set, wrapper() stops calling worker(), and Watcher() indicates queue.Empty and stopEvent and exits loop.
Finally main() calls terminate() of pool.
Sometimes processes done well, but sometimes hang. It's different each time.

You should put the code in try except block and catch a built-in exception KeyboardInterrupt see the example here Capture keyboardinterrupt

python threading not exit

I have a python script that uses threading, here is my code:
def main():
ip_list = get_ip_list() #['ip1', 'ip2'..., 'ipn'] thousands
for x in ip_list:
q.put(x)
threads = []
threads_num = 50
for x in range(threads_num):
w = Worker(q, stantard_config_to_dict, id_dict, logger,ip_position)
threads.append(w)
w.start()
for w in threads:
w.join()
logger.debug('End of main thread, exit')
print 'End of main thread, exit'
def run(self):
while 1:
try:
ip = self.queue.get(False)
self.logger.debug('%s get %s from queue, left %s in queue' % (self.getName(), ip, self.queue.qsize()))
self.get_inter_conf(ip)
except Queue.Empty:
self.logger.debug('queue is empty, exit')
break
self.logger.info("%s finished" % (self.getName()))
The string 'End of main thread' doesn't get printed right now, only when I change the number of items in ip_list it gets printed.
Why does this happen?

Handle multiple messages with Queue get()

Thanks to #user5402 for the previous solution.
I am trying to handle multiple messages that are queued up. Here is the code:
import sys
import socket
from multiprocessing import Process, Queue
UDP_ADDR = ("", 13000)
def send(m):
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.sendto(m, UDP_ADDR)
def receive(q):
buf = 1024
Sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
Sock.bind(UDP_ADDR)
while True:
(data, addr) = Sock.recvfrom(buf)
q.put(data)
In the client function I want to handle multiple messages, that have knock on affects.
def client():
q = Queue()
r = Process(target = receive, args=(q,))
r.start()
print "client loop started"
while True:
m = q.get()
print "got:", m
while m == "start":
print "started"
z = q.get()
if z == "stop":
return
print "loop ended"
r.terminate()
So when start is sent, it then goes into a while loop that is infinitely printing "started", and waiting for the stop message to come through. The above client code does not work.
Here is the code to start the function:
if __name__ == '__main__':
args = sys.argv
if len(args) > 1:
send(args[1])
else:
client()

You can write the client loop like this:
print "client loop started"
while True:
m = q.get()
print "waiting for start, got:", m
if m == "start":
while True:
try:
m = q.get(False)
except:
m = None
print "waiting for stop, got:", m
if m == "stop":
break
Based on your comments, this would be a better approach:
import sys
import socket
import Queue as Q
import time
from multiprocessing import Process, Queue
UDP_ADDR = ("", 13000)
def send(m):
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.sendto(m, UDP_ADDR)
def receive(q):
buf = 1024
Sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
Sock.bind(UDP_ADDR)
while True:
(data, addr) = Sock.recvfrom(buf)
q.put(data)
def doit():
# ... what the processing thread will do ...
while True:
print "sleeping..."
time.sleep(3)
def client():
q = Queue()
r = Process(target = receive, args=(q,))
r.start()
print "client loop started"
t = None # the processing thread
while True:
m = q.get()
if m == "start":
if t:
print "processing thread already started"
else:
t = Process(target = doit)
t.start()
print "processing thread started"
elif m == "stop":
if t:
t.terminate()
t = None
print "processing thread stopped"
else:
print "processing thread not running"
elif m == "quit":
print "shutting down"
if t:
t.terminate()
t = None # play it safe
break
else:
print "huh?"
r.terminate()
if __name__ == '__main__':
args = sys.argv
if len(args) > 1:
send(args[1])
else:
client()

How to streamline this script

I have this script and it does work it has 2 separate processes that spawn listener threads to kill the process when kill is sent to the listener via a pipe.
from multiprocessing import Process, Pipe
from threading import Thread
import time
subAlive = True
testAlive = True
def sub_listener(conn): #listens for kill from main
global subAlive
while True:
data = conn.recv()
if data == "kill":
subAlive = False #value for kill
break
def test_listener(conn): #listens for kill from main
global testAlive
while True:
data = conn.recv()
if data == "kill":
testAlive = False #value for kill
break
def subprocess(conn, threadNum):
t = Thread(target=sub_listener, args=(conn,))
count = 0
threadVal = threadNum
t.start()
while subAlive:
print "Thread %d Run number = %d" % (threadVal, count)
count = count + 1
def testprocess(conn, threadNum):
t = Thread(target=test_listener, args=(conn,))
count = 0
threadVal = threadNum
t.start()
while testAlive:
print "This is a different thread %d Run = %d" % (threadVal, count)
count = count + 1
sub_parent, sub_child = Pipe()
test_parent, test_child = Pipe()
runNum = int(raw_input("Enter a number: "))
threadNum = int(raw_input("Enter number of threads: "))
print "Starting threads"
for i in range(threadNum):
p = Process(target=subprocess, args=(sub_child, i))
p.start()
print "Subprocess started"
for i in range(threadNum):
p2 = Process(target=testprocess, args=(test_child, i))
p2.start()
print "Testproccess started"
print "Starting run"
time.sleep(runNum)
print "Terminating Subprocess run"
for i in range(threadNum):
sub_parent.send("kill") #sends kill to listener
print "Terminating Testprocess run"
for i in range(threadNum):
test_parent.send("kill") #sends kill to listener
p.join()
p2.join()
Id like to not need a separate listener function hard coded for every process I call. I was thinking about passing global variables when the thread is spawned. The global variables are really the only differences between the listener functions. Thanks guys!

You can access globals through the globals() dictionary.
>>> foo = 'value'
>>> def change(name):
... globals()[name] = 'changed'
...
>>> change('foo')
>>> foo
'changed'
but I would suggest:
alive = {}
def sub_listener(conn, key): #listens for kill from main
while True:
data = conn.recv()
if data == "kill":
alive[key] = False #value for kill
break
e.g.
from multiprocessing import Process, Pipe
from threading import Thread
import time
alive = {
'sub': 1,
'test': 1,
}
def listener_factory(key):
def listener(conn): #listens for kill from main
while True:
data = conn.recv()
if data == "kill":
alive[key] = False #value for kill
break
return listener
def process_factory(key):
listener = listener_factory(key)
def subprocess(conn, threadNum):
t = Thread(target=listener, args=(conn,))
count = 0
threadVal = threadNum
t.start()
while alive[key]:
print "Thread[%s] %d Run number = %d" % (key, threadVal, count)
count = count + 1
return subprocess
def main():
sub_parent, sub_child = Pipe()
test_parent, test_child = Pipe()
runNum = int(raw_input("Enter a number: "))
threadNum = int(raw_input("Enter number of threads: "))
print "Starting threads"
for i in range(threadNum):
p = Process(target=process_factory('sub'), args=(sub_child, i))
p.start()
print "Subprocess started"
for i in range(threadNum):
p2 = Process(target=process_factory('test'), args=(test_child, i))
p2.start()
print "Testproccess started"
print "Starting run"
time.sleep(runNum)
print "Terminating Subprocess run"
for i in range(threadNum):
sub_parent.send("kill") #sends kill to listener
print "Terminating Testprocess run"
for i in range(threadNum):
test_parent.send("kill") #sends kill to listener
p.join()
p2.join()
if __name__ == '__main__':
main()