I am trying to implement this tutorial on mutliprocessing in python, but when I tried to do my own task I get the following error:
Traceback (most recent call last):
>>> File "C:\Python27\lib\multiprocessing\queues.py", line 262, in _feed
send(obj)
IOError: [Errno 232] The pipe is being closed
Here is a reproducible example of what I am trying to do which gives the same error message:
from multiprocessing import Lock, Process, Queue, current_process
import time
class Testclass(object):
def __init__(self, x):
self.x = x
def toyfunction(testclass):
testclass.product = testclass.x * testclass.x
return testclass
def worker(work_queue, done_queue):
try:
for testclass in iter(work_queue.get, 'STOP'):
print(testclass.counter)
newtestclass = toyfunction(testclass)
done_queue.put(newtestclass)
except:
print('error')
return True
def main():
counter = 1
database = []
while counter <= 1000:
database.append(Testclass(3))
counter += 1
print(counter)
workers = 8
work_queue = Queue()
done_queue = Queue()
processes = []
start = time.clock()
counter = 1
for testclass in database:
testclass.counter = counter
work_queue.put(testclass)
counter += 1
print(counter)
print('items loaded')
for w in range(workers):
p = Process(target=worker, args=(work_queue, done_queue))
p.start()
processes.append(p)
work_queue.put('STOP')
for p in processes:
p.join()
done_queue.put('STOP')
print(time.clock()-start)
print("Done")
if __name__ == '__main__':
main()
I got around this by emptying the queue after using an event to exit the process gracefully:
self.event.set() #the process has a timer that checks for this to be set, then shuts itself down
while not self._q.empty(): #_q is a multiprocess.Queue object used to communicate inter-process
try:
self._q.get(timeout=0.001)
except:
pass
self._q.close()
When I add code that processes the done queue I no longer get the error. Here is working code:
from multiprocessing import Lock, Process, Queue, current_process
import time
class Testclass(object):
def __init__(self, x):
self.x = x
def toyfunction(testclass):
testclass.product = testclass.x * testclass.x
return testclass
def worker(work_queue, done_queue):
try:
for testclass in iter(work_queue.get, 'STOP'):
print(testclass.counter)
newtestclass = toyfunction(testclass)
done_queue.put(newtestclass)
except:
print('error')
return True
def main():
counter = 1
database = []
while counter <= 100:
database.append(Testclass(10))
counter += 1
print(counter)
workers = 8
work_queue = Queue()
done_queue = Queue()
processes = []
start = time.clock()
counter = 1
for testclass in database:
testclass.counter = counter
work_queue.put(testclass)
counter += 1
print(counter)
print('items loaded')
for w in range(workers):
p = Process(target=worker, args=(work_queue, done_queue))
p.start()
processes.append(p)
work_queue.put('STOP')
for p in processes:
p.join()
done_queue.put('STOP')
# added: process the done queue
newdatabase = []
for testclass in iter(done_queue.get, 'STOP'):
newdatabase.append(testclass)
print(time.clock()-start)
print("Done")
return(newdatabase)
if __name__ == '__main__':
database = main()
Related
the following code works fine -
import time
from concurrent.futures import Future, as_completed
from concurrent.futures.process import ProcessPoolExecutor
import multiprocessing as mp
from multiprocessing.managers import BaseManager
class Progress(object):
_target: int = 0
_progress: int = 0
def __init__(self):
self._target = 0
self._progress = 0
def completed(self, n):
self._progress += n
def progress(self):
return (self._progress/self._target) * 100
def set_target(self, n):
self._target = n
class ObjectManager(BaseManager):
pass
def dummy_worker(progress: Progress, cancel: mp.Event = None):
print("--> Worker started")
for i in range(10):
time.sleep(1)
progress.completed(1)
return 1
if __name__ == "__main__":
ObjectManager.register('Progress', Progress)
print('Starting manager')
with ObjectManager() as manager:
print('Manager started')
progress = manager.Progress()
progress.set_target(10)
with ProcessPoolExecutor() as pool:
f = pool.submit(dummy_worker, progress)
futures = [f]
for f in as_completed(futures):
print(f.result())
print(f'Progress: {progress.progress()}')
while the following gives me Condition objects should only be shared between processes through inheritance. i am not sure how to use Event in dummy_worker. the main goal is to pass some shared object and also have a way to cancel the worker. sorry for the long post, i wanted to give full code for clarity.
import time
from concurrent.futures import Future, as_completed
from concurrent.futures.process import ProcessPoolExecutor
import multiprocessing as mp
from multiprocessing.managers import BaseManager
class Progress(object):
_target: int = 0
_progress: int = 0
def __init__(self):
self._target = 0
self._progress = 0
def completed(self, n):
self._progress += n
def progress(self):
return (self._progress/self._target) * 100
def set_target(self, n):
self._target = n
class ObjectManager(BaseManager):
pass
def dummy_worker(progress: Progress, cancel: mp.Event = None):
print("--> Worker started")
for i in range(10):
time.sleep(1)
progress.completed(1)
return 1
if __name__ == "__main__":
ObjectManager.register('Progress', Progress)
cancel = mp.Event()
print('Starting manager')
with ObjectManager() as manager:
print('Manager started')
progress = manager.Progress()
progress.set_target(10)
with ProcessPoolExecutor() as pool:
f = pool.submit(dummy_worker, progress, cancel)
futures = [f]
for f in as_completed(futures):
print(f.result())
print(f'Progress: {progress.progress()}')
assume i am using python 3.5+.
Try the following changes:
from multiprocessing.managers import SyncManager
class ObjectManager(SyncManager):
pass
# use an Event() created by ObjectManager instance: cancel = manager.Event()
if __name__ == "__main__":
ObjectManager.register('Progress', Progress)
#cancel = mp.Event() # not this
print('Starting manager')
with ObjectManager() as manager:
print('Manager started')
progress = manager.Progress()
cancel = manager.Event() # but rather this
progress.set_target(10)
with ProcessPoolExecutor() as pool:
f = pool.submit(dummy_worker, progress, cancel)
futures = [f]
for f in as_completed(futures):
print(f.result())
print(f'Progress: {progress.progress()}')
Prints:
Starting manager
Manager started
--> Worker started
1
Progress: 100.0
Also, get rid of/change references to mp.Event.
In this easy example how can I access the value of a counter object while the process is still running?
import multiprocessing
import time
class Counter(object):
def __init__(self):
self.value = 0
def update(self):
self.value += 1
def job(Counter):
while True:
counter.update()
if __name__ == '__main__':
counter = Counter()
p = multiprocessing.Process(target=job,args=(counter,)
p.start()
time.sleep(10)
# I want to check the value of the counter object here
p.terminate()
You have to use multiprocessing.Queue() or multiprocessing.Pipe() to communicate between processes.
multiprocessing.Pipe() creates two endpoints conn_1, conn_2 and you have to use one of them in main process and second in subprocess.
Use poll() to check if there is something in pipe and then you can use recv() to receive data. (if you use directly recv() then it will block program till you send something to pipe).
Now you can use send() to send message with result.
Here I use conn_2 in job()
import multiprocessing
import time
class Counter(object):
def __init__(self):
self.value = 0
def update(self):
self.value += 1
def job(counter, conn):
while True:
counter.update()
if conn.poll():
print('job:', conn.recv())
conn.send(counter.value)
if __name__ == '__main__':
conn_1, conn_2 = multiprocessing.Pipe()
counter = Counter()
p = multiprocessing.Process(target=job, args=(counter, conn_2))
p.start()
time.sleep(2)
# I want to check the value of the counter object here
conn_1.send('give me result')
print('result:', conn_1.recv())
p.terminate()
Here I use conn_2 directly in class
import multiprocessing
import time
class Counter(object):
def __init__(self, conn):
self.conn = conn
self.value = 0
def update(self):
self.value += 1
if self.conn.poll(): # if message from main process
print('Counter:', self.conn.recv())
self.conn.send(self.value)
def job(counter):
while True:
counter.update()
if __name__ == '__main__':
conn_1, conn_2 = multiprocessing.Pipe()
counter = Counter(conn_2)
p = multiprocessing.Process(target=job, args=(counter,))
p.start()
time.sleep(2)
conn_1.send('give me result')
print('result:', conn_1.recv())
p.terminate()
You could consider attaching a debugger (such as the PyDev debugger, GDB or others) to the running process. You can then freeze the process with a breakpoint and inspect state.
My code is processing some parallel perforce tasks while showing a progress bar and letting user to terminate the job whenever he wants, the problem is when user clicks the close button the thread function is not being killed but the lock is released and the main UI thread is being unlocked.
The p4.run_sync() is not terminating when Cancel button is clicked.
def P4SyncLibrary(args, que):
syncType = args[0]
view = args[1]
p4 = P4CreateConnection(disable_tmp_cleanup=True)
try:
p4.run_sync(view)
except P4Exception:
for e in p4.errors:
print "SyncError: - %s" %e
p4.disconnect()
que.put(None)
class CreateJob(QtGui.QDialog):
def __init__(self, thread, args):
QtGui.QDialog.__init__(self)
self.ui=Ui_ProgressBar()
self.ui.setupUi(self)
self.ui.cancel.clicked.connect(self.closeEvent)
self.ui.cancel.setIcon(QtGui.QIcon(QtGui.QPixmap("%s/delete.xpm" %resources)))
self.threadControl = ThreadControl(thread=thread, args=args)
self.connect(self.threadControl, QtCore.SIGNAL("__updateProgressBar(int)"), self.__updateProgressBar)
self.threadControl.finished.connect(self.closeEvent)
self.threadControl.start()
#QtCore.pyqtSlot(int)
def __updateProgressBar(self,val):
self.ui.progressBar.setValue(val)
self.setWindowTitle("Processing: {0}%".format(val))
def closeEvent(self, QCloseEvent=None):
if self.threadControl.isRunning():
self.threadControl.stop()
self.threadControl.wait()
if QCloseEvent: QtGui.QDialog.closeEvent(self, QCloseEvent)
else: self.close()
def getResults(self):
return self.threadControl.resultDict
class ThreadControl(QtCore.QThread):
stopFlag = 0
def __init__(self, thread=None, args=None):
super(ThreadControl, self).__init__()
self.args = args
self.thread = thread
self.resultDict = []
def run(self):
threads = {}
queue = multiprocessing.Queue()
for arg in self.args:
process = multiprocessing.Process(target=self.thread, args=(arg, queue))
process.start()
threads[process] = 1 ## ACTIVE thread
# WAIT TILL ALL PROCESSES COMPLETE
completedThreads = 0
total = len(threads.keys())
while completedThreads != total:
if self.stopFlag:
for t in threads.keys():
if threads[t] == 1:
t.terminate()
t.join()
threads[t] = 0
completedThreads += 1
else:
for t in threads.keys():
if self.stopFlag: break ## Process threads termination
elif threads[t] == 1 and not t.is_alive():
threads[t] = 0
completedThreads += 1
self.resultDict.append(queue.get())
self.emit(QtCore.SIGNAL('__updateProgressBar(int)'),(completedThreads*100)/total)
sleep(0.5) ## Prevent CPU from overloading
def stop(self):
self.stopFlag=1
a job is being created using instance of CreateJob
CreateJob(thread=P4SyncLibrary, args=P4Libraries).exec_()
The only solution I could give is to pass p4 object to calling thread as argument so that p4 server connection can disconnect when user wants to cancel the job.
def P4SyncLibrary(p4, args, que):
syncType = args[0]
view = args[1]
try:
p4.run_sync(view)
except P4Exception:
for e in p4.errors:
print "SyncError: - %s" %e
que.put(None)
class ThreadControl(QtCore.QThread):
...
def run(self):
threads = {}
queue = multiprocessing.Queue()
for arg in self.args:
connection = P4CreateConnection(disable_tmp_cleanup=True)
if connection.connected():
process = multiprocessing.Process(target=self.thread, args=(connection, arg, queue))
process.start()
threads[process] = {
'isAlive': True,
'connection': connection
}
# WAIT TILL ALL PROCESSES COMPLETE
completedThreads = 0
total = len(threads.keys())
while completedThreads != total:
if self._stop:
for t in threads.keys():
if threads[t]['isAlive']:
threads[t]['connection'].disconnect()
t.terminate()
t.join()
threads[t]['isAlive'] = False
completedThreads += 1
else:
for t in threads.keys():
if self._stop: break ## Process threads termination
elif threads[t]['isAlive'] and not t.is_alive():
threads[t]['connection'].disconnect()
threads[t]['isAlive'] = False
completedThreads += 1
self.results.append(queue.get())
self.emit(QtCore.SIGNAL('__updateProgressBar(int)'),(completedThreads*100)/total)
sleep(0.5) ## Prevent CPU from overloading
I am trying my hands on python multiprocessing. I want a couple of processes which are independent to each other to run in parallel and as they return check if the process was successful or not using ApplyAsync.successful() utility. However when I call successful in the callback to my subprocess the script hangs.
import multiprocessing as mp
import time
result_map = {}
def foo_pool(x):
time.sleep(2)
print x
return x
result_list = []
def log_result(result):
print result_map[result].successful() #hangs
result_list.append(result)
def apply_async_with_callback():
pool = mp.Pool()
for i in range(10):
result_map[i] = pool.apply_async(foo_pool, args = (i, ), callback = log_result)
pool.close()
pool.join()
print(result_list)
if __name__ == '__main__':
apply_async_with_callback()
You don't need to check successful() because the callback is only called when the result was successful.
Following is the relevant code (multiprocessing/pool.py - AsyncResult)
def _set(self, i, obj):
self._success, self._value = obj
if self._callback and self._success: # <-----
self._callback(self._value) # <-----
self._cond.acquire()
try:
self._ready = True
self._cond.notify()
finally:
self._cond.release()
del self._cache[self._job]
I can't figure out what's wrong with the following python multiprocessing code. It does not terminate. Any suggestions will be highly appreciated.
from multiprocessing import Process, Queue, Lock
def hello(num, myqueue):
while True:
item = myqueue.get()
print 'Thread ', num, 'got', item
return
def put_on_queue(myqueue):
for i in range(10):
myqueue.put(i)
return
if __name__ == '__main__':
processes = []
myqueue = Queue()
for i in range(4):
proc = Process(target = hello, args = (i, myqueue))
proc.start()
processes.append(proc)
put_on_queue(myqueue)
for proc in processes:
proc.join()
-------------- EDIT -----------------
OK, so based on the comments I received, and some online help I revised my code as below. Still no luck :-(
def hello(num, myqueue):
while not exit_flag:
item = myqueue.get(False,5)
print 'Thread ', num, 'got', item
return
def put_on_queue(myqueue):
global exit_flag
for i in range(10):
myqueue.put(i)
while not myqueue.empty():
pass
exit_flag = 1
return
if __name__ == '__main__':
mylock = Lock()
processes = []
myqueue = Queue()
exit_flag = 0
for i in range(4):
proc = Process(target = hello, args = (i, myqueue))
#proc.daemon = True
proc.start()
processes.append(proc)
put_on_queue(myqueue)
for proc in processes:
proc.join()
There's an infinite loop inside your hello function.
You have to put a sentinent as a last value into the queue, check for it and break out of the loop.