I am having trouble with a sleep statement hanging my multithreading function. I want my function to go about it's buisness while the rest of the program runs. Here is a toy that recreates my problem:
import multiprocessing, sys, time
def f(icount, _sleepTime = 1):
for i in range(icount):
time.sleep(_sleepTime)
print(_sleepTime)
def main(args):
m = multiprocessing.Process(target = f, args=(4, ))
m.run()
# f should be sleeping for 1 second so this print statement should come first
print(m.is_alive())
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
can anyone explain why this code outputs:
1
1
1
1
False
instead of:
True
1
1
1
1
#
EDIT
#
I eventually want to run this function on a schedual, and test if it is running before I execute the function. This is an example:
import multiprocessing, sys, time
def f(icount, _sleepTime = 1):
for i in range(icount):
time.sleep(_sleepTime)
print(_sleepTime)
def main(args):
m = multiprocessing.Process(target = f, args=(4, ))
for i in range(15):
time.sleep(.5)
if not m.is_alive():
# m.start throws an error after first run
m.run()
print("{}".format(m.is_alive()))
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
Use start and join instead of run:
import multiprocessing, sys, time
def f(icount, _sleepTime = 1):
for i in range(icount):
time.sleep(_sleepTime)
print(_sleepTime)
def main(args):
m = multiprocessing.Process(target = f, args=(4, ))
m.start()
# f should be sleeping for 1 second so this print statement should come first
print(m.is_alive())
m.join()
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
#
EDIT
#
Again, use start and join instead of run:
import multiprocessing, sys, time
def f(icount, _sleepTime = 1):
for i in range(icount):
time.sleep(_sleepTime)
print(_sleepTime)
def create_process():
return multiprocessing.Process(target = f, args=(4, ))
def main(args):
m = create_process()
m.start()
for i in range(15):
time.sleep(.5)
if not m.is_alive():
# m.start throws an error after first run
print("restarting")
m.join()
m = create_process()
m.start()
print("{}".format(m.is_alive()))
m.join()
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
Related
I am trying to play around with multiprocessing and I would like to communicate between Python's main thread and a subprocess with a Queue. Here is a quick test code I wrote that should get periodically some results generated by the subprocess:
from multiprocessing import Process, Queue
import time
def calculate(queue):
n = 0
while n < 10:
n += 1
queue.put(n)
time.sleep(1)
queue.put(0)
def queue_getter(queue):
executing = True
while executing:
while queue.qsize():
n = queue.get()
print(n)
if n == 0:
executing = False
time.sleep(0.1)
print('done')
queue = Queue()
p = Process(target=calculate, args=(queue,))
p.start()
queue_getter(queue)
p.join()
print('DONE')
This program just hangs forever, while replacing Process with threading.Thread gives the expected result:
1
2
3
4
5
6
7
8
9
10
0
done
DONE
How to make Process behave the same way as Thread in this situation?
Your program works fine on POSIX (UNIX-like) systems.
However, for it to work properly on ms-windows and macOS, you will need to put the program itself inside a main block, so the file can be imported without side effects.
This is due to the way multiprocessing has to work on ms-windows and macOS. Read the programming guidelines for multiprocessing.
Modify your code like this:
from multiprocessing import Process, Queue
import time
def calculate(queue):
n = 0
while n < 10:
n += 1
queue.put(n)
time.sleep(1)
queue.put(0)
def queue_getter(queue):
executing = True
while executing:
while queue.qsize():
n = queue.get()
print(n)
if n == 0:
executing = False
time.sleep(0.1)
print("done")
if __name__ == "__main__":
queue = Queue()
p = Process(target=calculate, args=(queue,))
p.start()
queue_getter(queue)
p.join()
print("DONE")
Here's a simplified and more robust approach which is (almost) functionally identical to the OP's original except that is does not print the zero:
from multiprocessing import Manager
from concurrent.futures import ProcessPoolExecutor
import time
def calculate(q):
for n in range(1, 11):
q.put(n)
time.sleep(1)
q.put(0)
def queue_getter(q):
while (n := q.get()):
print(n)
def main():
with Manager() as manager:
q = manager.Queue()
with ProcessPoolExecutor() as executor:
executor.submit(calculate, q)
queue_getter(q)
if __name__ == '__main__':
main()
I am running a multiprocessing task in python, how can I timeout a function after 60seconds.
What I have done is shown in the snippet below:
import multiprocessing as mp
from multiprocessing import Pool
from multiprocessing import Queue
def main():
global f
global question
global queue
queue = Queue()
processes = []
question = [16,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,17,18,19,21,20,23]
cores=5
loww=0
chunksize = int((len(question)-loww)/cores)
splits = []
for i in range(cores):
splits.append(loww+1+((i)*chunksize))
splits.append(len(question)+1)
print("",splits)
args = []
for i in range(cores):
a=[]
arguments = (i, splits[i], splits[i+1])
a.append(arguments)
args.append(a)
print(args)
p = Pool(cores)
p.map(call_process, args)
p.close()
p.join
def call_process(args):
## end this whole block if it is taking more than 1 minutes
starttime = time.time()
lower=args[0][1]
upper=args[0][2]
for x in range(lower,upper):
if time.time() >= starttime + 60: break
a = question[x-1]
try:
pass
# a lot of functions is called and returned here
except:
continue
#write item to file
print('a = ',a)
return a
main()
I want to ensure that the call_process() method does not run for more than a minute for a particular value. Currently, I am using if time.time() >= starttime + 60: break which would not work effectively as I have different functions and things happening in the try and except block. What can I do?
I use multiprocessing lib to test python multi process, but I meet some problems. I have test code 1:
import multiprocessing
def test(name):
print 'processing.....'
tmp = 0
for i in xrange(1000000000):
tmp += i
print 'process done'
if __name__ == '__main__':
pools = multiprocessing.Pool()
for i in xrange(2):
pools.apply_async(test)
pools.close()
pools.join()
result is:
processing
processing
done
done
Code 2:
import multiprocessing
class Test:
def test(name):
print 'processing.....'
tmp = 0
for i in xrange(1000000000):
tmp += i
print 'process done'
if __name__ == '__main__':
t = Test()
pools = multiprocessing.Pool()
for i in xrange(4):
pools.apply_async(t.test)
pools.close()
pools.join()
this result is nothing, this pools don't call t.test! I can't understand what happended. Why is this?
instead of using pool, you can simply collect the jobs in a list:
import multiprocessing
class Test(multiprocessing.Process):
def run(self):
print 'processing.....'
tmp = 0
for i in xrange(10):
tmp += i
print 'process done'
return 1
if __name__ == '__main__':
jobs = []
for i in range(5):
t = Test()
jobs.append(t)
t.start()
the list jobs will be able to tell you if the process has finished or not, ultimately giving you the same effect as using pool.
if you wanna make sure that all jobs are done:
if __name__ == '__main__':
jobs = []
for i in range(5):
t = Test()
jobs.append(t)
t.start()
not_done = any(job.is_alive() for job in jobs)
while not_done:
not_done = any(job.is_alive() for job in jobs)
print 'job all done'
I am trying my hands on python multiprocessing. I want a couple of processes which are independent to each other to run in parallel and as they return check if the process was successful or not using ApplyAsync.successful() utility. However when I call successful in the callback to my subprocess the script hangs.
import multiprocessing as mp
import time
result_map = {}
def foo_pool(x):
time.sleep(2)
print x
return x
result_list = []
def log_result(result):
print result_map[result].successful() #hangs
result_list.append(result)
def apply_async_with_callback():
pool = mp.Pool()
for i in range(10):
result_map[i] = pool.apply_async(foo_pool, args = (i, ), callback = log_result)
pool.close()
pool.join()
print(result_list)
if __name__ == '__main__':
apply_async_with_callback()
You don't need to check successful() because the callback is only called when the result was successful.
Following is the relevant code (multiprocessing/pool.py - AsyncResult)
def _set(self, i, obj):
self._success, self._value = obj
if self._callback and self._success: # <-----
self._callback(self._value) # <-----
self._cond.acquire()
try:
self._ready = True
self._cond.notify()
finally:
self._cond.release()
del self._cache[self._job]
How to give a variable from _While.py to __scheduler.py in this multithreading Szenario?
I tried so much stuff but nothing seems to be working.
Thread.py
from multiprocessing import Process
import _While
import _Scheduler
if __name__ == '__main__':
p1 = Process(target=_While.main)
p1.start()
p2 = Process(target=_Scheduler.main)
p2.start()
_While.py
import time
def main():
while True:
print "while"
time.sleep(0.5)
"""getting button status"""
"""giving button status to _Scheudler._scheduler"""
__Scheduler.py
import logging
import time
from apscheduler.scheduler import Scheduler
from _While import """button status"""
def _scheduler():
print "scheduler"
while """button status"""==True:
print "Button is pressed"
time.sleep(0.5)
def main():
logging.basicConfig()
scheduler = Scheduler(standalone=True)
scheduler.add_interval_job(_scheduler, seconds=2)
scheduler.start()
if __name__ == '__main__':
main()
Solution:
Thread.py
from multiprocessing import Process, Value, Array
import time
import _While
import _Scheduler
if __name__ == '__main__':
num = Value('d', 0.0)
arr = Array('i', range(10))
p1 = Process(target=_While.main, args=(num, arr))
p1.start()
p2 = Process(target=_Scheduler.main, args=(num, arr))
p2.start()
p1.join()
p2.join()
print num.value
_While
import time
def main(num, arr):
while True:
print "while"
num.value = 1
time.sleep(10)
"""getting button status"""
"""giving button status to _Scheudler._scheduler"""
__Scheduler.py
import logging
from apscheduler.scheduler import Scheduler
def _scheduler(num, arr):
while True:
print num.value
if num.value == 1:
print "mongo"
num.value = 0
break
def main(num, arr):
logging.basicConfig()
scheduler = Scheduler(standalone=True)
scheduler.add_interval_job(_scheduler, args=(num, arr), seconds=2)
scheduler.start()
if __name__ == '__main__':
main()
The only problem left is that I can't use Value without using Array
Create an instance of multiprocessing.Value in Threads.py just before you create p1 and p2, then pass the instance of Value as args to both p1 and p2 and change the main() method of _While.py and _Scheduler.py to accept the new Value parameter.
Similar to how it is done here http://docs.python.org/2/library/multiprocessing.html#sharing-state-between-processes
You could also use Queues or Pipes as suggested by Euegene C. an example can be found here http://docs.python.org/2/library/multiprocessing.html#exchanging-objects-between-processes