In this easy example how can I access the value of a counter object while the process is still running?
import multiprocessing
import time
class Counter(object):
def __init__(self):
self.value = 0
def update(self):
self.value += 1
def job(Counter):
while True:
counter.update()
if __name__ == '__main__':
counter = Counter()
p = multiprocessing.Process(target=job,args=(counter,)
p.start()
time.sleep(10)
# I want to check the value of the counter object here
p.terminate()
You have to use multiprocessing.Queue() or multiprocessing.Pipe() to communicate between processes.
multiprocessing.Pipe() creates two endpoints conn_1, conn_2 and you have to use one of them in main process and second in subprocess.
Use poll() to check if there is something in pipe and then you can use recv() to receive data. (if you use directly recv() then it will block program till you send something to pipe).
Now you can use send() to send message with result.
Here I use conn_2 in job()
import multiprocessing
import time
class Counter(object):
def __init__(self):
self.value = 0
def update(self):
self.value += 1
def job(counter, conn):
while True:
counter.update()
if conn.poll():
print('job:', conn.recv())
conn.send(counter.value)
if __name__ == '__main__':
conn_1, conn_2 = multiprocessing.Pipe()
counter = Counter()
p = multiprocessing.Process(target=job, args=(counter, conn_2))
p.start()
time.sleep(2)
# I want to check the value of the counter object here
conn_1.send('give me result')
print('result:', conn_1.recv())
p.terminate()
Here I use conn_2 directly in class
import multiprocessing
import time
class Counter(object):
def __init__(self, conn):
self.conn = conn
self.value = 0
def update(self):
self.value += 1
if self.conn.poll(): # if message from main process
print('Counter:', self.conn.recv())
self.conn.send(self.value)
def job(counter):
while True:
counter.update()
if __name__ == '__main__':
conn_1, conn_2 = multiprocessing.Pipe()
counter = Counter(conn_2)
p = multiprocessing.Process(target=job, args=(counter,))
p.start()
time.sleep(2)
conn_1.send('give me result')
print('result:', conn_1.recv())
p.terminate()
You could consider attaching a debugger (such as the PyDev debugger, GDB or others) to the running process. You can then freeze the process with a breakpoint and inspect state.
Related
with some help I could run a process in python, Now I wan't to share a value betwenn the two tasks. I can set the value inside the init, but I can't change it inside the run method.
And by the way: how to kill the process when the main process stops?
from multiprocessing import Process, Value
import serial
import time
class P(Process):
def __init__(self, num):
num.value = 15
super(P, self).__init__()
def run(self):
while True:
num.value = num.value + 1
print("run simple process")
time.sleep(0.5)
def main():
while True:
print("run main")
print (num.value)
time.sleep(2.5)
if __name__ == "__main__":
num = Value('d', 0.0)
p = P(num)
p.start()
#p.join()
main()
In your simplified case you just passed num value upon initialization time.
To be able to access that value in other process's methods - set it as a state of the process:
class P(Process):
def __init__(self, num):
self.num = num
self.num.value = 15
super(P, self).__init__()
def run(self):
while True:
self.num.value += 1
print("run simple process")
time.sleep(0.5)
For a more "serious" cases - consider using Managers and Synchronization primitives.
This is my current code, the main issue is I use Semphore to control the output of two process, but it seems like the Semphore does not change globaly, i.e. when process "producer" change the Semphore to 2 the process "consumer" still think the Semphore is zero , which cause it to wait forever.
from multiprocessing import Process, Semaphore, Queue
import time
from random import random
buffer = Queue(10)
empty = Semaphore(2)
full = Semaphore(0)
class Consumer(Process):
def run(self):
global buffer, empty, full
while True:
time.sleep(4)
print(full)
full.acquire()
buffer.get()
print('Consumer get')
time.sleep(1)
empty.release()
class Producer(Process):
def run(self):
global buffer, empty, full
while True:
empty.acquire()
print ('Producer put ')
time.sleep(1)
full.release()
buffer.put(1)
print(full)
if __name__ == '__main__':
p = Producer()
c = Consumer()
p.daemon = c.daemon = True
p.start()
c.start()
p.join()
c.join()
print ('Ended!')
and the output is
Producer put
<Semaphore(value=1)>
Producer put
<Semaphore(value=2)>
<Semaphore(value=0)>
I don't know what should I do to let "consumer" process detect the change.
Your two processes have both their own copy of both the semaphores, because each process runs the whole code in the script when it is instantiated.
You must move the semaphores and queue definitions inside the if __name__ == '__main__': and pass the instances of the semaphores to the Producer and Consumer constructors so that they both use the same instance of the three objects.
from multiprocessing import Process, Semaphore, Lock, Queue
import time
from random import random
class Consumer(Process):
def __init__(self, empty, full, buffer):
super(Consumer, self).__init__()
self.empty = empty
self.full = full
self.buffer = buffer
def run(self):
while True:
time.sleep(4)
print("Consumer: {}".format(self.full), flush=True)
print("Consumer: buf {}".format(self.buffer.qsize()), flush=True)
self.full.acquire()
self.buffer.get()
print('Consumer get', flush=True)
time.sleep(1)
self.empty.release()
class Producer(Process):
def __init__(self, empty, full, buffer):
super(Process, self).__init__()
self.empty = empty
self.full = full
self.buffer = buffer
def run(self):
while True:
self.empty.acquire()
print ('Producer put ', flush=True)
self.buffer.put('a') #<<<<<<<<<<< you forgot this in your code. If the queue is empty, get() will block on the consumer
time.sleep(1)
self.full.release()
print(self.full, flush=True)
if __name__ == '__main__':
buffer = Queue(10)
empty = Semaphore(2)
full = Semaphore(0)
p = Producer(empty, full, buffer)
c = Consumer(empty, full, buffer)
p.daemon = c.daemon = True
p.start()
c.start()
p.join()
c.join()
print ('Ended!')
#coding:utf-8
import sys
import time
import os
import multiprocessing
class Worker(object):
def __init__(self):
self.progress = 0
self.task_info = None
def init(self):
pass
def status(self):
pass
def set_task_info(self, task_info):
self.task_info = task_info
def run(self, worker_status_meta_dict):
print multiprocessing.current_process()
print "process is %d" % self.progress
while self.progress < 5:
self.progress = self.progress +1
worker_status_meta_dict['state'] = 0
worker_status_meta_dict['status'] = "running"
time.sleep(2)
worker_status_meta_dict['state'] = 1
worker_status_meta_dict['status'] = "succeeded"
print "bavscan worker finished..."
if __name__ == "__main__":
worker = Worker()
worker_process_dict = multiprocessing.Manager().dict()
process = multiprocessing.Process(target=Worker.run, args=(worker, worker_process_dict))
process.start()
time.sleep(60)
This is a simple demo for python multiprocess.
The main process invoke the Worker.run method in a subprocess with multiprocessing.Process.
When run it in wondows 7, the main process will lauch two subprocess.
I find the problem in the "Python27\Lib\multiprocessing__init__.py"
def Manager():
'''
Returns a manager associated with a running server process
The managers methods such as `Lock()`, `Condition()` and `Queue()`
can be used to create shared objects.
'''
from multiprocessing.managers import SyncManager
m = SyncManager()
m.start()
return m
m.start() will lauch a subprocess to start the manager.
I have a little doubt if one could solve my issue, and create successful communication between threads.
First example and this is how it should be working, but does not work well:
import Queue,threading,time
class th(threading.Thread):
def __init__(self,q):
threading.Thread.__init__(self)
self.q = q
self.t = time
def run(self):
for i in range(5):
self.q.put(i)
self.t.sleep(0.5) # <----------
self.q.put('end')
class main(object):
def __init__(self):
self.q = Queue.Queue()
self.thread = th(self.q)
self.thread.setDaemon(True)
self.thread.run()
self.call()
def call(self):
while True:
recv = self.q.get();
if recv == 'end':
break
else:
print recv
if __name__ == '__main__':
root = main()
root.call()
In this example, all printed at the same time:
0,1,2,3,4
Second example:
import Queue,threading,time
class th(threading.Thread):
def __init__(self,q):
threading.Thread.__init__(self);
self.q = q
self.t = time
def run(self):
for i in range(5):
self.q.put(i) # <------ no sleep()
self.q.put('end')
class main(object):
def __init__(self):
self.q = Queue.Queue()
self.thread = th(self.q)
self.thread.setDaemon(True)
self.thread.run()
self.call()
def call(self):
while True:
recv = self.q.get()
if recv == 'end':
break
else:
print recv
if __name__ == '__main__':
root = main()
root.call()
the code is printed as it has to
0,
1
2
3
4
one to one
is there any way that the sleep function in the same way?
You don't want to call the run method on a thread directly. Call start instead, which will kick off the child thread, which will in turn run the run method.
Your current code is essentially single threaded, since the run call does the work of the child thread in the parent instead. The child thread is never actually started! (You're also calling your main.call method twice, which I'd expect to block or raise an exception, but that's a separate issue.)
sorry, it was something very simple, really simple, just had to replace
self.thread.run()
by
self.threat.start()
I've been looking into a way to directly change variables in a running module.
What I want to achieve is that a load test is being run and that I can manually adjust the call pace or whatsoever.
Below some code that I just created (not-tested e.d.), just to give you an idea.
class A():
def __init__(self):
self.value = 1
def runForever(self):
while(1):
print self.value
def setValue(self, value):
self.value = value
if __name__ == '__main__':
#Some code to create the A object and directly apply the value from an human's input
a = A()
#Some parallelism or something has to be applied.
a.runForever()
a.setValue(raw_input("New value: "))
Edit #1: Yes, I know that now I will never hit the a.setValue() :-)
Here is a multi-threaded example. This code will work with the python interpreter but not with the Python Shell of IDLE, because the raw_input function is not handled the same way.
from threading import Thread
from time import sleep
class A(Thread):
def __init__(self):
Thread.__init__(self)
self.value = 1
self.stop_flag = False
def run(self):
while not self.stop_flag:
sleep(1)
print(self.value)
def set_value(self, value):
self.value = value
def stop(self):
self.stop_flag = True
if __name__ == '__main__':
a = A()
a.start()
try:
while 1:
r = raw_input()
a.set_value(int(r))
except:
a.stop()
The pseudo code you wrote is quite similar to the way Threading / Multiprocessing works in python. You will want to start a (for example) thread that "runs forever" and then instead of modifying the internal rate value directly, you will probably just send a message through a Queue that gives the new value.
Check out this question.
Here is a demonstration of doing what you asked about. I prefer to use Queues to directly making calls on threads / processes.
import Queue # !!warning. if you use multiprocessing, use multiprocessing.Queue
import threading
import time
def main():
q = Queue.Queue()
tester = Tester(q)
tester.start()
while True:
user_input = raw_input("New period in seconds or (q)uit: ")
if user_input.lower() == 'q':
break
try:
new_speed = float(user_input)
except ValueError:
new_speed = None # ignore junk
if new_speed is not None:
q.put(new_speed)
q.put(Tester.STOP_TOKEN)
class Tester(threading.Thread):
STOP_TOKEN = '<<stop>>'
def __init__(self, q):
threading.Thread.__init__(self)
self.q = q
self.speed = 1
def run(self):
while True:
# get from the queue
try:
item = self.q.get(block=False) # don't hang
except Queue.Empty:
item = None # do nothing
if item:
# stop when requested
if item == self.STOP_TOKEN:
break # stop this thread loop
# otherwise check for a new speed
try:
self.speed = float(item)
except ValueError:
pass # whatever you like with unknown input
# do your thing
self.main_code()
def main_code(self):
time.sleep(self.speed) # or whatever you want to do
if __name__ == '__main__':
main()