I'm trying to run 2 separate processes in my python application. So I have code like this:
from multiprocessing import Process
def f1():
while 1:
print('Hello')
def f2():
while 1:
print('Goodbye')
def main():
p1 = Process(target=f1, args=())
p1.start()
p1.join()
p2 = Process(target=f2, args=())
p2.start()
p2.join()
if __name__ == '__main__':
main()
This code does nothing on my machine, it doesn't produce any output. I thought initially that maybe it was an IDE-related problem, but it's the same on both my IDEs, PyScripter and IDLE.
Any ideas, why this doesn't print anything?
How about using Queue?
from multiprocessing import Process, Queue
def f1(q):
while 1:
q.put('Hello')
def f2(q):
while 1:
q.put('Goodbye')
def main():
q = Queue()
p1 = Process(target=f1, args=(q,))
p1.start()
p2 = Process(target=f2, args=(q,))
p2.start()
while True:
try:
print q.get()
except:
break
if __name__ == '__main__':
main()
You should save it and run outside the IDE:
C:\> python multi.py
then it infinitely prints out Hello. You should change your main to see both Hello and Goodbye:
def main():
p1 = Process(target=f1, args=())
p2 = Process(target=f2, args=())
p1.start()
p2.start()
p1.join()
p2.join()
Then you have a little happy race condition that constantly prints out GHoodbyeello because both processes use the same stdout resource concurrently.
Related
hellow,
please some help.
i want to take variables when using repeating statement.
Actually in my code, there are so many variables and function to handle variables.
so i have to use multiprocess for some reason, but it's doesn't work for what i want.
below is simple code,
please help me.
from multiprocessing import Process, Manager
import time
def a(final_list):
c=0
while True:
c += 1
final_list.append(c)
time.sleep(1)
print(final_list)
def b(final_list):
while True:
print(final_list[-1])
time.sleep(1)
if __name__ == '__main__':
manager = Manager()
final_list = []
final_list = manager.list()
#print(a)
p1 = Process(target=a, args=(final_list,))
p2 = Process(target=b, args=(final_list,))
p1.start()
time.sleep(3)
p2.start()
I think you forgot to use join() for the processes. try this:
from multiprocessing import Process, Manager
import time
def a(final_list):
c=0
while True:
c += 1
final_list.append(c)
time.sleep(1)
print(final_list)
def b(final_list):
while True:
print(final_list[-1])
time.sleep(1)
if __name__ == '__main__':
with Manager() as manager:
final_list = manager.list()
p1 = Process(target=a, args=(final_list,))
p2 = Process(target=b, args=(final_list,))
p1.start()
time.sleep(3)
p2.start()
p1.join()
p2.join()
I have two functions and needed the return values to proceed with the further part of the script...but currently my code giving only the output of the first function...
import multiprocessing
def gm(name):
h = "Good Morning"+str(name)
qout.put(h)
def sal(name):
k="Hi "+str(name)
qout.put(k)
if __name__ == '__main__':
qout = multiprocessing.Queue()
p1 = multiprocessing.Process(target=gm, args=("ashin",))
p2 = multiprocessing.Process(target=sal, args=("ashin",))
p1.start()
p2.start()
p1.join()
p2.join()
result = qout.get()
#output - "Good Morning ashin"
#required output - "Good Morning ashin" & "Hi ashin"
Appreciate your help......
qout.get() gets you the first element from queue. I do not know the bigger picture of what you're are trying to achieve, but you can get all elements from queue like in the following.
from multiprocessing import Process, Queue
def gm(name):
h = "Good Morning"+str(name)
qout.put(h)
def sal(name):
k="Hi "+str(name)
qout.put(k)
if __name__ == '__main__':
qout = Queue()
p1 = Process(target=gm, args=("ashin",))
p2 = Process(target=sal, args=("ashin",))
p1.start()
p2.start()
p1.join()
p2.join()
list1 = []
while not qout.empty():
list1.append(qout.get())
temp = list(map(str, list1))
print(" & ".join(temp))
output
Hi ashin & Good Morningashin
Instead of managing your own output queue, just use the latest Python 3 concurrency features:
from concurrent.futures import as_completed, ProcessPoolExecutor
def gm(name):
return f'Good Morning {name}'
def sal(name):
return f'Hi {name}'
if __name__ == '__main__':
with ProcessPoolExecutor() as exe:
futures = [exe.submit(x, 'ashin') for x in (gm, sal)]
for future in as_completed(futures):
print(future.result())
I'm making use of multiprocessing but I get this error "MongoClient opened before fork." for every process. I did some research and concluded that I'm now creating multiple MongoClients (one per subprocess). But I didn't find a real solution. Every process is making use of MongoDB connection (I'm using pymongo as connector). Can someone help me?
Code:
def func1():
while True:
col1.insert_one({...})
...
def func2():
while True:
col2.insert_one({...})
...
if __name__ == "__main__":
# MongoDB
myclient = pymongo.MongoClient("mongodb://localhost:27017/")
mydb = myclient["testdb"]
col1 = mydb["col1"]
col2 = mydb["col2"]
# Multiproccesing
p1 = Process(target=func1)
p2 = Process(target=func2)
p1.start()
p2.start()
p1.join()
p2.join()
Have each process open their own MongoDB connection(s).
Heed the warning in get_mongo_client(); if you want something that's safe to call from wherever, you'll need to "tag" _mongo_client with the PID of the current process and discard the object if it has the wrong PID.
_mongo_client = None # Global per process
def get_mongo_client():
# Make sure not to call this within the master process, or things
# will break again.
global _mongo_client
if _mongo_client is None:
_mongo_client = pymongo.MongoClient("mongodb://localhost:27017/")
return _mongo_client
def get_mongo_col(collection, database="testdb"):
client = get_mongo_client()
return client[database][collection]
def func1():
col1 = get_mongo_col("col1")
while True:
col1.insert_one({})
# ...
def func2():
col2 = get_mongo_col("col2")
while True:
col2.insert_one({})
# ...
def main():
# Multiproccesing
p1 = Process(target=func1)
p2 = Process(target=func2)
p1.start()
p2.start()
p1.join()
p2.join()
if __name__ == "__main__":
main()
Based on this pretty useful tutorial I have tried to make a simple implementation of Python multiprocessing to measure its effectivity. The modules multi1, multi2, multi3 contain an ODE integration and exporting the calculated values in a csv (it does not matter, they are here for a script to do something).
import multiprocessing
import multi1
import multi2
import multi3
import time
t0 = time.time()
if __name__ == '__main__':
p1 = multiprocessing.Process(target = multi1.main(), args=())
p2 = multiprocessing.Process(target = multi2.main(), args=())
p3 = multiprocessing.Process(target = multi3.main(), args=())
p1.start()
p2.start()
p3.start()
p1.join()
p2.join()
p3.join()
t1 = time.time()
multi1.main()
multi2.main()
multi3.main()
t2 = time.time()
print t1-t0
print t2-t1
The problem is that the printed times are equal, so the multiprocessing didn't speed up the process. Why?
You called main in the main thread, and passed the return value (probably None) as the target, so no actual work is done in your worker processes. Remove the call parens, so you pass the function itself without calling it, e.g.:
p1 = multiprocessing.Process(target=multi1.main, args=())
p2 = multiprocessing.Process(target=multi2.main, args=())
p3 = multiprocessing.Process(target=multi3.main, args=())
This is the same basic problem seen in the threaded case.
I have 2 process running and I want them to complete before further down command executes (at the end of script it prints out that the program has ended). How can I make sure the process completes before printing out that it has ended?
from multiprocessing import Process
import datetime
class foo:
def fun1():
do sthn
def fun2():
do sthn
ob = foo()
if __name__ == '__main__':
p1 = Process(target = ob.fun1)
p1.start()
p2 = Process(target = ob.fun2)
p2.start()
endTime=datetime.datetime.now()
print 'Program Ending time is: ', endTime
You would use the .join() method, which blocks until the process is complete.
p1.join()
p2.join()