Develop Reference

Join timeout in multiprocessing

I have a dummy example, I want to apply multiprocessing in it. Consider a scenario where you have a stream of numbers(which I call frame) incoming one by one. And I want to assign it to any single process that is available currently. So I am creating 4 processes that are running a while loop, seeing if any element in queue, than apply function on it.
The problem is that when I join it, it gets stuck in any while loop, even though I close the while loop before it. But somehow it gets stuck inside it.
Code:
# step 1, 4 processes
import multiprocessing as mp
import os
import time
class MpListOperations:
def __init__(self):
self.results_queue = mp.Manager().Queue()
self.frames_queue = mp.Manager().Queue()
self.flag = mp.Manager().Value(typecode='b',value=True)
self.list_nums = list(range(0,5000))
def process_list(self):
print(f"Process id {os.getpid()} started")
while self.flag.value:
# print(self.flag.value)
if self.frames_queue.qsize():
self.results_queue.put(self.frames_queue.get()**2)
def create_processes(self, no_of_processes = mp.cpu_count()):
print("Creating Processes")
self.processes = [mp.Process(target=self.process_list) for _ in range(no_of_processes)]
def start_processes(self):
print(f"starting processes")
for process in self.processes:
process.start()
def join_process(self):
print("Joining Processes")
while True:
if not self.frames_queue.qsize():
self.flag.value=False
print("JOININNG HERE")
for process in self.processes:
exit_code = process.join()
print(exit_code)
print("BREAKING DONE")
break
def stream_frames(self):
print("Streaming Frames")
for frame in self.list_nums:
self.frames_queue.put(frame)
if __name__=="__main__":
start = time.time()
mp_ops = MpListOperations()
mp_ops.create_processes()
mp_ops.start_processes()
mp_ops.stream_frames()
mp_ops.join_process()
print(time.time()-start)
Now if I add a timeout parameter in join, even 0, i.e exit_code = process.join(0) it works. I want to understand in this scenario, if this code is correct, what should be the value of timeout? Why is it working with timeout and not without it? What is the proper way to implement multiprocessing with it?

If you look at the documentation for a managed queue you will see that the qsize method only returns an approximate size. I would therefore not use it for testing when all the items have been taken of the frames queue. Presumably you want to let the processes run until all frames have been processed. The simplest way I know would be to put N sentinel items on the frames queue after the actual frames have been put where N is the number of processes getting from the queue. A sentinel item is a special value that cannot be mistaken for an actual frame and signals to the process that there are no more items for it to get from the queue (i.e. a quasi end-of-file item). In this case we can use None as the sentinel items. Each process then just continues to do get operations on the queue until it sees a sentinel item and then terminates. There is therefore no need for the self.flag attribute.
Here is the updated and simplified code. I have made some other minor changes that have been commented:
import multiprocessing as mp
import os
import time
class MpListOperations:
def __init__(self):
# Only create one manager process:
manager = mp.Manager()
self.results_queue = manager.Queue()
self.frames_queue = manager.Queue()
# No need to convert range to a list:
self.list_nums = range(0, 5000)
def process_list(self):
print(f"Process id {os.getpid()} started")
while True:
frame = self.frames_queue.get()
if frame is None: # Sentinel?
# Yes, we are done:
break
self.results_queue.put(frame ** 2)
def create_processes(self, no_of_processes = mp.cpu_count()):
print("Creating Processes")
self.no_of_processes = no_of_processes
self.processes = [mp.Process(target=self.process_list) for _ in range(no_of_processes)]
def start_processes(self):
print("Starting Processes")
for process in self.processes:
process.start()
def join_processes(self):
print("Joining Processes")
for process in self.processes:
# join returns None:
process.join()
def stream_frames(self):
print("Streaming Frames")
for frame in self.list_nums:
self.frames_queue.put(frame)
# Put sentinels:
for _ in range(self.no_of_processes):
self.frames_queue.put(None)
if __name__== "__main__":
start = time.time()
mp_ops = MpListOperations()
mp_ops.create_processes()
mp_ops.start_processes()
mp_ops.stream_frames()
mp_ops.join_processes()
print(time.time()-start)
Prints:
Creating Processes
Starting Processes
Process id 28 started
Process id 29 started
Streaming Frames
Process id 33 started
Process id 31 started
Process id 38 started
Process id 44 started
Process id 42 started
Process id 45 started
Joining Processes
2.3660173416137695
Note for Windows
I have modified method start_processes to temporarily set attribute self.processes to None:
def start_processes(self):
print("Starting Processes")
processes = self.processes
# Don't try to pickle list of processes:
self.processes = None
for process in processes:
process.start()
# Restore attribute:
self.processes = processes
Otherwise under Windows we get a pickle error trying to serialize/deserialize a list of processes containing two or more multiprocessing.Process instances. The error is "TypeError: cannot pickle 'weakref' object." This can be demonstrated with the following code where we first try to pickle a list of 1 process and then a list of 2 processes:
import multiprocessing as mp
import os
class Foo:
def __init__(self, number_of_processes):
self.processes = [mp.Process(target=self.worker) for _ in range(number_of_processes)]
self.start_processes()
self.join_processes()
def start_processes(self):
processes = self.processes
for process in self.processes:
process.start()
def join_processes(self):
for process in self.processes:
process.join()
def worker(self):
print(f"Process id {os.getpid()} started")
print(f"Process id {os.getpid()} ended")
if __name__== "__main__":
foo = Foo(1)
foo = Foo(2)
Prints:
Process id 7540 started
Process id 7540 ended
Traceback (most recent call last):
File "C:\Booboo\test\test.py", line 26, in <module>
foo = Foo(2)
File "C:\Booboo\test\test.py", line 7, in __init__
self.start_processes()
File "C:\Booboo\test\test.py", line 13, in start_processes
process.start()
File "C:\Program Files\Python38\lib\multiprocessing\process.py", line 121, in start
self._popen = self._Popen(self)
File "C:\Program Files\Python38\lib\multiprocessing\context.py", line 224, in _Popen
return _default_context.get_context().Process._Popen(process_obj)
File "C:\Program Files\Python38\lib\multiprocessing\context.py", line 327, in _Popen
return Popen(process_obj)
File "C:\Program Files\Python38\lib\multiprocessing\popen_spawn_win32.py", line 93, in __init__
reduction.dump(process_obj, to_child)
File "C:\Program Files\Python38\lib\multiprocessing\reduction.py", line 60, in dump
ForkingPickler(file, protocol).dump(obj)
TypeError: cannot pickle 'weakref' object
Process id 18152 started
Process id 18152 ended
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "C:\Program Files\Python38\lib\multiprocessing\spawn.py", line 116, in spawn_main
exitcode = _main(fd, parent_sentinel)
File "C:\Program Files\Python38\lib\multiprocessing\spawn.py", line 126, in _main
self = reduction.pickle.load(from_parent)
EOFError: Ran out of input

The target loop is stuck in the get() method of your loop. This is because multiple processes could see that the queue wasn't empty, but only 1 of them was able to get the last item. The remaining processes are waiting for the next item to be available from the queue.
You might need to add a Lock when you are reading the size of the Queue object And getting the object of that queue.
Or alternatively, you avoid reading the size of the queue by simply using the queue.get() method with a timeout that allows us to check the flag regularly
import queue
TIMEOUT = 1 # seconds
class MpListOperations:
#[...]
def process_list(self):
print(f"Process id {os.getpid()} started")
previous = self.flag.value
while self.flag.value:
try:
got = self.frames_queue.get(timeout=TIMEOUT)
except queue.Empty:
pass
else:
print(f"Gotten {got}")
self.results_queue.put(got**2)
_next = self.flag.value
if previous != _next:
print(f"Flag change: {_next}")
$ python ./test_mp.py
Creating Processes
starting processes
Process id 36566 started
Streaming Frames
Process id 36565 started
Process id 36564 started
Process id 36570 started
Process id 36567 started
Gotten 0
Process id 36572 started
Gotten 1
Gotten 2
Gotten 3
Process id 36579 started
Gotten 4
Gotten 5
Gotten 6
Process id 36583 started
Gotten 7
# [...]
Gotten 4997
Joining Processes
Gotten 4998
Gotten 4999
JOININNG HERE
Flag change: False
Flag change: False
Flag change: False
Flag change: False
Flag change: False
Flag change: False
Flag change: False
Flag change: False
Exit code : None
Exit code : None
Exit code : None
Exit code : None
Exit code : None
Exit code : None
Exit code : None
Exit code : None
BREAKING DONE
1.4375360012054443
Alternatively, using a multiprocessing.Pool object:
def my_func(arg):
time.sleep(0.002)
return arg**2
def get_input():
for i in range(5000):
yield i
time.sleep(0.001)
if __name__=="__main__":
start = time.time()
mp_pool = mp.Pool()
result = mp_pool.map(my_func, get_input())
mp_pool.close()
mp_pool.join()
print(len(result))
print(f"Duration: {time.time()-start}")
Giving:
$ python ./test_mp.py
5000
Duration: 6.847279787063599

python mutiprocessing on windows using shared values

I am trying to gain an insight into using multiprocessing with python. I have an example of using shared values for Unix but I cannot get a simple educational example to work on Windows 10.
I have the code below running ok on Windows but with the call updating the shared value commented out in foo().
What is my problem please?
import multiprocessing as mp
def foo(q):
#global shared_num
q.put('hello')
#shared_num.value = 777
if __name__ == '__main__':
global shared_num
mp.set_start_method('spawn')
shared_num = mp.Value('d', 0)
lock = mp.Lock()
q = mp.Queue()
p = mp.Process(target=foo, args=(q,))
p.start()
p.join()
print(q.get(), " ",shared_num.value)
#print(q.get(), " ")
If I run the code below with the foo() setting the shared value I get:
Traceback (most recent call last):
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2032.0_x64__qbz5n2kfra8p0\lib\multiprocessing\process.py", line 315, in _bootstrap
self.run()
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2032.0_x64__qbz5n2kfra8p0\lib\multiprocessing\process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "C:\Users\ken38\OneDrive\Projects\Python\GH_Pi\python\ms_mp.py", line 6, in foo
shared_num.value = 777
NameError: name 'shared_num' is not defined
hello 0.0

Michael Butscher actually answered this with his comment. But as I could not flag that as an answer I thought I would show the corrected code as an answer.
This does highlight difference if you try and test simple examples on Windows. Linux based examples may not work an Windows.
This code worked on both Windows and Debian (Rpi).
import multiprocessing as mp
def foo(q, shared_num, lock):
#global shared_num
q.put('hello')
with lock:
shared_num.value = 777
if __name__ == '__main__':
global shared_num
mp.set_start_method('spawn')
shared_num = mp.Value('d', 0)
lock = mp.Lock()
q = mp.Queue()
p = mp.Process(target=foo, args=(q, shared_num, lock,))
p.start()
p.join()
print(q.get(), " ",shared_num.value)
#print(q.get(), " ")

An error while using locks in multiprocessing in Python

I'm a beginner learning python and ran into some issues while using locks during multiprocessing.
I get an exit code 0 and the right answer but still have some sort of error message which I really don't fully understand. Here's The code I've written-
import time
import multiprocessing
def deposit(balance):
for i in range(100):
time.sleep(0.01)
lck.acquire()
balance.value += 1
lck.release()
def withdraw(balance):
for i in range(100):
time.sleep(0.01)
lck.acquire()
balance.value -= 1
lck.release()
if __name__ == '__main__':
balance = multiprocessing.Value('i', 200)
lck = multiprocessing.Lock()
d = multiprocessing.Process(target=deposit, args=(balance,))
w = multiprocessing.Process(target=withdraw, args=(balance,))
d.start()
w.start()
d.join()
w.join()
print(balance.value)
and here's the error I get
`Process Process-1:
Traceback (most recent call last):
File "C:\Users\rahul\AppData\Local\Programs\Python\Python39\lib\multiprocessing\process.py", line
315, in _bootstrap
self.run()
File "C:\Users\rahul\AppData\Local\Programs\Python\Python39\lib\multiprocessing\process.py", line
108, in run
self._target(*self._args, **self._kwargs)
File "C:\Users\rahul\PycharmProjects\pythonProject\LearningPython.py", line 10, in deposit
lck.acquire()
NameError: name 'lck' is not defined
Process Process-2:
Traceback (most recent call last):
File "C:\Users\rahul\AppData\Local\Programs\Python\Python39\lib\multiprocessing\process.py", line
315, in _bootstrap
self.run()
File "C:\Users\rahul\AppData\Local\Programs\Python\Python39\lib\multiprocessing\process.py", line
108, in run
self._target(*self._args, **self._kwargs)
File "C:\Users\rahul\PycharmProjects\pythonProject\LearningPython.py", line 17, in withdraw
lck.acquire()
NameError: name 'lck' is not defined
200
Process finished with exit code 0

The problem here is that lck is out of scope of your child processes. Global variables aren't shared across processes. Try passing the lock into the processes.
Alternatively use threads as suggested in kahn's answer. They are much friendlier and still work fine in this case.
import time
import multiprocessing
def deposit(balance,lck):
for i in range(100):
time.sleep(0.01)
lck.acquire()
balance.value += 1
lck.release()
def withdraw(balance,lck):
for i in range(100):
time.sleep(0.01)
lck.acquire()
balance.value -= 1
lck.release()
if __name__ == '__main__':
balance = multiprocessing.Value('i', 200)
lck = multiprocessing.Lock()
d = multiprocessing.Process(target=deposit, args=(balance,lck))
w = multiprocessing.Process(target=withdraw, args=(balance,lck))
d.start()
w.start()
d.join()
w.join()
print(balance.value)

Because deposit and withdraw run in other processes.In their view,the process is not __main__,so the if statement is not executed and lck is not defined.
Try Run
import os
import multiprocessing
def deposit(balance):
print(os.getpid(),__name__)
def withdraw(balance):
print(os.getpid(),__name__)
if __name__ == '__main__':
print(os.getpid(), __name__)
balance = multiprocessing.Value('i', 200)
lck = multiprocessing.Lock()
d = multiprocessing.Process(target=deposit, args=(balance,))
w = multiprocessing.Process(target=withdraw, args=(balance,))
d.start()
w.start()
d.join()
w.join()
In my case ,it shows
19604 __main__
33320 __mp_main__
45584 __mp_main__
Your code can run if you put lck = multiprocessing.Lock() outside if.But I'm sure it's not what you want.
You should use threading instead of multiprocess in this case,and have a look at difference between multi-thread and multi-process.

Asyncio in corroutine RuntimeError: no running event loop

I'm writing multi-process code, which runs perfectly in Python 3.7. Yet I want one of the parallel process to execute an IO process take stakes for ever using AsyncIO i order to get better performance, but have not been able to get it to run.
Ubuntu 18.04, Python 3.7, AsyncIO, pipenv (all pip libraries installed)
The method in particular runs as expected using multithreading, which is what I want to replace with AsyncIO.
I have googled and tried looping in the main() function and now only in the intended cor-routine, have looked at examples and read about this new Async way of getting things down and no results so far.
The following is the app.py code which is esecuted: python app.py
import sys
import traceback
import logging
import asyncio
from config import DEBUG
from config import log_config
from <some-module> import <some-class>
if DEBUG:
logging.config.dictConfig(log_config())
else:
logging.basicConfig(
level=logging.DEBUG, format='%(relativeCreated)6d %(threadName)s %(message)s')
logger = logging.getLogger(__name__)
def main():
try:
<some> = <some-class>([
'some-data1.csv',
'some-data2.csv'
])
<some>.run()
except:
traceback.print_exc()
pdb.post_mortem()
sys.exit(0)
if __name__ == '__main__':
asyncio.run(main())
Here is the code where I have the given class defined
_sql_client = SQLServer()
_blob_client = BlockBlobStore()
_keys = KeyVault()
_data_source = _keys.fetch('some-data')
# Multiprocessing
_manager = mp.Manager()
_ns = _manager.Namespace()
def __init__(self, list_of_collateral_files: list) -> None:
#timeit
def _get_filter_collateral(self, ns: mp.managers.NamespaceProxy) -> None:
#timeit
def _get_hours(self, ns: mp.managers.NamespaceProxy) -> None:
#timeit
def _load_original_bids(self, ns: mp.managers.NamespaceProxy) -> None:
#timeit
def _merge_bids_with_hours(self, ns: mp.managers.NamespaceProxy) -> None:
#timeit
def _get_collaterial_per_month(self, ns: mp.managers.NamespaceProxy) -> None:
#timeit
def _calc_bid_per_path(self) -> None:
#timeit
def run(self) -> None:
The method containing the async code is here:
def _get_filter_collateral(self, ns: mp.managers.NamespaceProxy) -> None:
all_files = self._blob_client.download_blobs(self._list_of_blob_files)
_all_dfs = pd.DataFrame()
async def read_task(file_: str) -> None:
nonlocal _all_dfs
df = pd.read_csv(StringIO(file_.content))
_all_dfs = _all_dfs.append(df, sort=False)
tasks = []
loop = asyncio.new_event_loop()
for file_ in all_files:
tasks.append(asyncio.create_task(read_task(file_)))
loop.run_until_complete(asyncio.wait(tasks))
loop.close()
_all_dfs['TOU'] = _all_dfs['TOU'].map(lambda x: 'OFFPEAK' if x == 'OFF' else 'ONPEAK')
ns.dfs = _all_dfs
And the method that calls the particular sequence and and this async method is:
def run(self) -> None:
extract = []
extract.append(mp.Process(target=self._get_filter_collateral, args=(self._ns, )))
extract.append(mp.Process(target=self._get_hours, args=(self._ns, )))
extract.append(mp.Process(target=self._load_original_bids, args=(self._ns, )))
# Start the parallel processes
for process in extract:
process.start()
# Await for database process to end
extract[1].join()
extract[2].join()
# Merge both database results
self._merge_bids_with_hours(self._ns)
extract[0].join()
self._get_collaterial_per_month(self._ns)
self._calc_bid_per_path()
self._save_reports()
self._upload_data()
These are the errors I get:
Process Process-2:
Traceback (most recent call last):
File "<some-path>/.pyenv/versions/3.7.4/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
self.run()
File "<some-path>/.pyenv/versions/3.7.4/lib/python3.7/multiprocessing/process.py", line 99, in run
self._target(*self._args, **self._kwargs)
File "<some-path>/src/azure/application/utils/lib.py", line 10, in timed
result = method(*args, **kwargs)
File "<some-path>/src/azure/application/caiso/main.py", line 104, in _get_filter_collateral
tasks.append(asyncio.create_task(read_task(file_)))
File "<some-path>/.pyenv/versions/3.7.4/lib/python3.7/asyncio/tasks.py", line 350, in create_task
loop = events.get_running_loop()
RuntimeError: no running event loop
<some-path>/.pyenv/versions/3.7.4/lib/python3.7/multiprocessing/process.py:313: RuntimeWarning: coroutine '<some-class>._get_filter_collateral.<locals>.read_task' was never awaited
traceback.print_exc()
RuntimeWarning: Enable tracemalloc to get the object allocation traceback
DEBUG Calculating monthly collateral...
Traceback (most recent call last):
File "app.py", line 25, in main
caiso.run()
File "<some-path>/src/azure/application/utils/lib.py", line 10, in timed
result = method(*args, **kwargs)
File "<some-path>/src/azure/application/caiso/main.py", line 425, in run
self._get_collaterial_per_month(self._ns)
File "<some-path>/src/azure/application/utils/lib.py", line 10, in timed
result = method(*args, **kwargs)
File "<some-path>/src/azure/application/caiso/main.py", line 196, in _get_collaterial_per_month
credit_margin = ns.dfs
File "<some-path>/.pyenv/versions/3.7.4/lib/python3.7/multiprocessing/managers.py", line 1122, in __getattr__
return callmethod('__getattribute__', (key,))
File "<some-path>/.pyenv/versions/3.7.4/lib/python3.7/multiprocessing/managers.py", line 834, in _callmethod
raise convert_to_error(kind, result)
AttributeError: 'Namespace' object has no attribute 'dfs'
> <some-path>/.pyenv/versions/3.7.4/lib/python3.7/multiprocessing/managers.py(834)_callmethod()
-> raise convert_to_error(kind, result)
(Pdb)

As it seems from the Traceback log it is look like you are trying to add tasks to not running event loop.
/.pyenv/versions/3.7.4/lib/python3.7/multiprocessing/process.py:313:
RuntimeWarning: coroutine
'._get_filter_collateral..read_task' was never
awaited
The loop was just created and it's not running yet, therefor the asyncio unable to attach tasks to it.
The following example will reproduce the same results, adding tasks and then trying to await for all of them to finish:
import asyncio
async def func(num):
print('My name is func {0}...'.format(num))
loop = asyncio.get_event_loop()
tasks = list()
for i in range(5):
tasks.append(asyncio.create_task(func(i)))
loop.run_until_complete(asyncio.wait(tasks))
loop.close()
Results with:
Traceback (most recent call last):
File "C:/tmp/stack_overflow.py", line 42, in <module>
tasks.append(asyncio.create_task(func(i)))
File "C:\Users\Amiram\AppData\Local\Programs\Python\Python37-32\lib\asyncio\tasks.py", line 324, in create_task
loop = events.get_running_loop()
RuntimeError: no running event loop
sys:1: RuntimeWarning: coroutine 'func' was never awaited
Nonetheless the solution is pretty simple, you just need to add the tasks to the created loop - instead of asking the asyncio to go it.
The only change is needed in the following line:
tasks.append(asyncio.create_task(func(i)))
Change the creation of the task from the asyncio to the newly created loop, you are able to do it because this is your loop unlike the asynio which is searching for a running one.
So the new line should look like this:
tasks.append(loop.create_task(func(i)))
Another solution could be running an async function and create the tasks there for example (Because that loop is already running now the asyncio enable to attach tasks to it):
import asyncio
async def func(num):
print('Starting func {0}...'.format(num))
await asyncio.sleep(0.1)
print('Ending func {0}...'.format(num))
loop = asyncio.get_event_loop()
async def create_tasks_func():
tasks = list()
for i in range(5):
tasks.append(asyncio.create_task(func(i)))
await asyncio.wait(tasks)
loop.run_until_complete(create_tasks_func())
loop.close()
This simple change will results with:
Starting func 0...
Starting func 1...
Starting func 2...
Starting func 3...
Starting func 4...
Ending func 0...
Ending func 2...
Ending func 4...
Ending func 1...
Ending func 3...

Use asyncio.ensure_future instead. See https://docs.python.org/3/library/asyncio-future.html#asyncio.ensure_future

Python Multiprocess issues process not starting

I wrote this code, I want to have a main thread that starts multiple subprocesses that spawn a listener thread to wait for a kill message subprocess works but testprocess does not run there are no errors any ideas?
from multiprocessing import Process, Pipe
from threading import Thread
import time
Alive = True
def listener_thread(conn): #listens for kill from main
global Alive
while True:
data = conn.recv()
if data == "kill":
Alive = False #value for kill
break
def subprocess(conn):
t = Thread(target=listener_thread, args=(conn,))
count = 0
t.start()
while Alive:
print "Run number = %d" % count
count = count + 1
def testprocess(conn):
t = Thread(target=listner_thread, args=(conn,))
count = 0
t.start()
while Alive:
print "This is a different thread run = %d" % count
count = count + 1
parent_conn, child_conn = Pipe()
p = Process(target=subprocess, args=(child_conn,))
p2 = Process(target=testprocess, args=(child_conn,))
runNum = int(raw_input("Enter a number: "))
p.start()
p2.start()
time.sleep(runNum)
parent_conn.send("kill") #sends kill to listener thread to tell them when to stop
p.join()
p2.join()

A typo in testprocess makes the function to quit early.
listner_thread should be listener_thread.
If you comment out subprocess related code and run the code, you will see following error:
Process Process-1:
Traceback (most recent call last):
File "/usr/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/usr/lib/python2.7/multiprocessing/process.py", line 114, in run
self._target(*self._args, **self._kwargs)
File "t.py", line 25, in testprocess
t = Thread(target=listner_thread, args=(conn,))
NameError: global name 'listner_thread' is not defined