We want to know if the executor has reached max_workers. Is there a simple way to find out how many workers are running in the following code?
import concurrent.futures
def dummy_process(arg_a, arg_b):
print("ml_process", arg_a, arg_b)
time.sleep(5)
executor = concurrent.futures.ProcessPoolExecutor(max_workers=2)
def main():
while True:
executor.submit(dummy_process, "test_a", "test_b")
if __name__ == "__main__":
main()
use try except
import concurrent.futures
def dummy_process(arg_a, arg_b):
print("ml_process", arg_a, arg_b)
time.sleep(5)
executor = concurrent.futures.ProcessPoolExecutor(max_workers=2)
def main():
while True:
try:
executor.submit(dummy_process, "test_a", "test_b")
except:
print("Pool Executer is Full")
if __name__ == "__main__":
main()
I am really frustrated. Why doesn't Python's multiprocessing.apply_async() actually START the process when a queue object is passed as an argument or a part of an argument?
This code works as expected:
#! /usr/bin/env python3
import multiprocessing
import queue
import time
def worker(var):
while True:
print("Worker {}".format(var))
time.sleep(2)
pool = multiprocessing.Pool(20)
m = multiprocessing.Manager()
q = queue.Queue()
for i in range(20):
pool.apply_async(worker, (i,))
print("kicked off workers")
pool.close()
pool.join()
But just by passing queue q, nothing happens when you run it now:
#! /usr/bin/env python3
import multiprocessing
import queue
import time
def worker(var,q):
while True:
print("Worker {}".format(var))
time.sleep(2)
pool = multiprocessing.Pool(20)
m = multiprocessing.Manager()
q = queue.Queue()
for i in range(20):
pool.apply_async(worker, (i,q))
print("kicked off workers")
pool.close()
pool.join()
Again; super frustrating. What the hell is going on? What am I doing wrong?
When you want to share a Queue between processes, you have to create a proxy for one with multiprocessing.managers.SyncManager.Queue.
import multiprocessing
import time
def worker(var, q):
while True:
print("Worker {}".format(var))
time.sleep(2)
if __name__ == '__main__': # Be sure to include this.
pool = multiprocessing.Pool(20)
mgr = multiprocessing.Manager()
q = mgr.Queue() # Create a shared queue.Queue object.
for i in range(20):
pool.apply_async(worker, (i,q))
print("kicked off workers")
pool.close()
print('joining pool')
pool.join()
print('done')
I have implemented a parser like this,
import multiprocessing
import time
def foo(i):
try:
# some codes
except Exception, e:
print e
def worker(i):
foo(i)
time.sleep(i)
return i
if __name__ == "__main__":
pool = multiprocessing.Pool(processes=4)
result = pool.map_async(worker, range(15))
while not result.ready():
print("num left: {}".format(result._number_left))
time.sleep(1)
real_result = result.get()
pool.close()
pool.join()
My parser actually finishes all the processes but the results are not available ie, it's still inside the while loop and printing num left : 2. How I stop this? And I don't want the value of real_result variable.
I'm running Ubuntu 14.04, python 2.7
Corresponding part of my code looks like,
async_args = ((date, kw_dict) for date in dates)
pool = Pool(processes=4)
no_rec = []
def check_for_exit(msg):
print msg
if last_date in msg:
print 'Terminating the pool'
pool.terminate()
try:
result = pool.map_async(parse_date_range, async_args)
while not result.ready():
print("num left: {}".format(result._number_left))
sleep(1)
real_result = result.get(5)
passed_dates = []
for x, y in real_result:
passed_dates.append(x)
if y:
no_rec.append(y[0])
# if last_date in passed_dates:
# print 'Terminating the pool'
# pool.terminate()
pool.close()
except:
print 'Pool error'
pool.terminate()
print traceback.format_exc()
finally:
pool.join()
My bet is that you have faulty parse_date_range,
which causes a worker process to terminate without producing any result or py exception.
Probably libc's exit is called by a C module/lib due to a realy nasty error.
This code reproduces the infinite loop you observe:
import sys
import multiprocessing
import time
def parse_date_range(i):
if i == 5:
sys.exit(1) # or raise SystemExit;
# other exceptions are handled by the pool
time.sleep(i/19.)
return i
if __name__ == "__main__":
pool = multiprocessing.Pool(4)
result = pool.map_async(parse_date_range, range(15))
while not result.ready():
print("num left: {}".format(result._number_left))
time.sleep(1)
real_result = result.get()
pool.close()
pool.join()
Hope this'll help.
I have a python pool of processes , if an exception occurs in any one of the process i want to exit the execution of the pool
I have joined all the processes in the pool, so the join waits for every process to finish.
If i raise sys.exit(1) inside the target function the system goes on infinite wait because the join is still waiting for process to complete.
How can exit the execution while using join in the code
from multiprocessing import Pool
import time
import sys
def printer(ip):
try:
for _ in xrange(5):
print ip+str(_)
time.sleep(1.0)
except Exception as e:
print e
sys.exit(2)
def test():
pool = Pool(processes=2)
for i in ["hello",5]:
result = pool.apply_async(printer,(i,))
pool.close()
pool.join()
print "good bye"
test()
Just return to the parent process the status of the operation and use a callback to react to failures.
import time
from multiprocessing import Pool
def printer(ip):
try:
for _ in xrange(5):
print ip+str(_)
time.sleep(1.0)
return True
except Exception as e:
print e
return False
class Worker():
def __init__(self):
self.pool = Pool(processes=2)
def callback(self, result):
if not result:
print "Error raised in child process. Terminating.."
self.pool.terminate()
def do_job(self):
for i in ["hello", 5]:
self.pool.apply_async(printer, (i,), callback=self.callback)
self.pool.close()
self.pool.join()
print "good bye"
def test():
w = Worker()
w.do_job()
I have a script and I want one function to run at the same time as the other.
The example code I have looked at:
import threading
def MyThread (threading.thread):
# doing something........
def MyThread2 (threading.thread):
# doing something........
MyThread().start()
MyThread2().start()
I am having trouble getting this working. I would prefer to get this going using a threaded function rather than a class.
This is the working script:
from threading import Thread
class myClass():
def help(self):
os.system('./ssh.py')
def nope(self):
a = [1,2,3,4,5,6,67,78]
for i in a:
print i
sleep(1)
if __name__ == "__main__":
Yep = myClass()
thread = Thread(target = Yep.help)
thread2 = Thread(target = Yep.nope)
thread.start()
thread2.start()
thread.join()
print 'Finished'
You don't need to use a subclass of Thread to make this work - take a look at the simple example I'm posting below to see how:
from threading import Thread
from time import sleep
def threaded_function(arg):
for i in range(arg):
print("running")
sleep(1)
if __name__ == "__main__":
thread = Thread(target = threaded_function, args = (10, ))
thread.start()
thread.join()
print("thread finished...exiting")
Here I show how to use the threading module to create a thread which invokes a normal function as its target. You can see how I can pass whatever arguments I need to it in the thread constructor.
There are a few problems with your code:
def MyThread ( threading.thread ):
You can't subclass with a function; only with a class
If you were going to use a subclass you'd want threading.Thread, not threading.thread
If you really want to do this with only functions, you have two options:
With threading:
import threading
def MyThread1():
pass
def MyThread2():
pass
t1 = threading.Thread(target=MyThread1, args=[])
t2 = threading.Thread(target=MyThread2, args=[])
t1.start()
t2.start()
With thread:
import thread
def MyThread1():
pass
def MyThread2():
pass
thread.start_new_thread(MyThread1, ())
thread.start_new_thread(MyThread2, ())
Doc for thread.start_new_thread
I tried to add another join(), and it seems worked. Here is code
from threading import Thread
from time import sleep
def function01(arg,name):
for i in range(arg):
print(name,'i---->',i,'\n')
print (name,"arg---->",arg,'\n')
sleep(1)
def test01():
thread1 = Thread(target = function01, args = (10,'thread1', ))
thread1.start()
thread2 = Thread(target = function01, args = (10,'thread2', ))
thread2.start()
thread1.join()
thread2.join()
print ("thread finished...exiting")
test01()
Python 3 has the facility of Launching parallel tasks. This makes our work easier.
It has for thread pooling and Process pooling.
The following gives an insight:
ThreadPoolExecutor Example
import concurrent.futures
import urllib.request
URLS = ['http://www.foxnews.com/',
'http://www.cnn.com/',
'http://europe.wsj.com/',
'http://www.bbc.co.uk/',
'http://some-made-up-domain.com/']
# Retrieve a single page and report the URL and contents
def load_url(url, timeout):
with urllib.request.urlopen(url, timeout=timeout) as conn:
return conn.read()
# We can use a with statement to ensure threads are cleaned up promptly
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
# Start the load operations and mark each future with its URL
future_to_url = {executor.submit(load_url, url, 60): url for url in URLS}
for future in concurrent.futures.as_completed(future_to_url):
url = future_to_url[future]
try:
data = future.result()
except Exception as exc:
print('%r generated an exception: %s' % (url, exc))
else:
print('%r page is %d bytes' % (url, len(data)))
Another Example
import concurrent.futures
import math
PRIMES = [
112272535095293,
112582705942171,
112272535095293,
115280095190773,
115797848077099,
1099726899285419]
def is_prime(n):
if n % 2 == 0:
return False
sqrt_n = int(math.floor(math.sqrt(n)))
for i in range(3, sqrt_n + 1, 2):
if n % i == 0:
return False
return True
def main():
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
for number, prime in zip(PRIMES, executor.map(is_prime, PRIMES)):
print('%d is prime: %s' % (number, prime))
if __name__ == '__main__':
main()
You can use the target argument in the Thread constructor to directly pass in a function that gets called instead of run.
Did you override the run() method? If you overrided __init__, did you make sure to call the base threading.Thread.__init__()?
After starting the two threads, does the main thread continue to do work indefinitely/block/join on the child threads so that main thread execution does not end before the child threads complete their tasks?
And finally, are you getting any unhandled exceptions?
the simple way to implement multithread process using threading
code snippet for the same
import threading
#function which takes some time to process
def say(i):
time.sleep(1)
print(i)
threads = []
for i in range(10):
thread = threading.Thread(target=say, args=(i,))
thread.start()
threads.append(thread)
#wait for all threads to complete before main program exits
for thread in threads:
thread.join()