Python class object Sharing between worker processes - python

Here is my code. I am not getting why it is giving the Error: AttributeError: 'AutoProxy[MySharedClass]' object has no attribute 'dict_'
import time
import multiprocessing
from multiprocessing import Process, Manager
import multiprocessing.managers as manager
class MySharedClass(object):
def __init__(self):
self.dict_ = {}
self.dict_['one'] = 1
self.dict_['two'] = 2
class MyManager(manager.BaseManager):
pass
def test_process(param_object):
print"Child Process:"
print param_object.dict_
param_object.dict_['three'] = 3
print param_object.dict_
return param_object
def main_process():
MyManager.register("MySharedClass", MySharedClass)
my_manager = MyManager()
my_manager.start()
param_object = my_manager.MySharedClass()
print"Main process:"
print param_object.dict_
time.sleep(.2)
proc = Process(target=test_process, args=(param_object,))
proc.start()
print"Process created with id %s" %proc.pid
time.sleep(10)
print "Main process End:"
print param_object.dict_
main_process()

Related

Python multiprocessing exits without exception when accessing shared value. Is this normal?

I am trying to share a string between processes with multiprocessing.Value(c_wchar_p, ""). Program exits unexpectedly without exception with code (0xC0000005). I am guessing that problem is somehow caused by main process trying to access memory of child process. Issue occurs only when ctypes c_char_p/c_wchar_p are used. Thread works fine. Is this expected beheavor?
Steps to reproduce:
import multiprocessing
import threading
import ctypes
import time
class ProcessExample:
def __init__(self, proc: bool):
self.shared_bool = multiprocessing.Value(ctypes.c_bool, False)
self.shared_string = multiprocessing.Value(ctypes.c_wchar_p, "one")
self.shared_byte = multiprocessing.Value(ctypes.c_char_p, b"one")
if proc:
self.th_proc = multiprocessing.Process(target=self.target, args=(self.shared_bool, self.shared_string, self.shared_byte))
else:
self.th_proc = threading.Thread(target=self.target, args=(self.shared_bool, self.shared_string, self.shared_byte))
self.th_proc.start()
#staticmethod
def target(shared_bool, shared_string, shared_byte):
shared_bool.value = True
shared_string.value = "two"
shared_byte.value = b"two"
def print_values(self):
self.th_proc.join()
print(self.shared_bool.value)
print(self.shared_string.value) # Exits
print(self.shared_byte.value) # Exits
if __name__ == "__main__":
# example = ProcessExample(False) # Works
example = ProcessExample(True)
time.sleep(1)
example.print_values()
Output
True
Process finished with exit code -1073741819 (0xC0000005)
I think the problem is in multiprocessing.Value(ctypes.c_wchar_p, "one") and multiprocessing.Value(ctypes.c_char_p, b"one"). To have string/byte string initialized correctly, use multiprocessing.Array:
import multiprocessing
import threading
import ctypes
import time
class ProcessExample:
def __init__(self, proc: bool):
self.shared_bool = multiprocessing.Value(ctypes.c_bool, False)
self.shared_string = multiprocessing.Array(ctypes.c_wchar, "one")
self.shared_byte = multiprocessing.Array(ctypes.c_char, b"one")
if proc:
self.th_proc = multiprocessing.Process(
target=self.target,
args=(self.shared_bool, self.shared_string, self.shared_byte),
)
else:
self.th_proc = threading.Thread(
target=self.target,
args=(self.shared_bool, self.shared_string, self.shared_byte),
)
self.th_proc.start()
#staticmethod
def target(shared_bool, shared_string, shared_byte):
shared_bool.value = True
shared_string[:] = "two"
shared_byte[:] = b"two"
def print_values(self):
self.th_proc.join()
print(self.shared_bool.value)
print(self.shared_string[:])
print(self.shared_byte[:])
if __name__ == "__main__":
example = ProcessExample(True)
time.sleep(1)
example.print_values()
Prints:
True
two
b'two'

Keep static class members in python multiprocessing

I'm trying to keep a "static" defined multiprocessing Queue through multiple Processes, but it appears that this context is not copied to the new spawned process. Is there a way to keep them without storing them to derived process classes (so without self.q = A.q)?
main.py
from class_b import B
if __name__ == "__main__":
b = B()
b.start()
while True:
pass
class_a.py
from multiprocessing import Process, Queue
class A(Process):
q = Queue()
def __init__(self) -> None:
super().__init__(daemon=True)
class_b.py
from multiprocessing import Process
from class_a import A
class B(Process):
def __init__(self):
super().__init__(daemon=True)
print(A.q)
def run(self):
print(A.q)
console
<multiprocessing.queues.Queue object at 0x000001F77851B280>
<multiprocessing.queues.Queue object at 0x0000023C420C2580>
When you import from class_a.py to access A.q, then so does multiprocessing in its own process. Then there will be two copies. You should create it as a local in "main" and pass it into B.
from class_b import B
from multiprocessing import Queue
if __name__ == "__main__":
q = Queue()
b = B(q)
b.start()
while True:
pass
Then make B store that reference for itself:
from multiprocessing import Process
class B(Process):
def __init__(self, q):
super().__init__(daemon=True)
print(q)
self.q = q
def run(self):
print(self.q)

python Class Object Sharing between Processes created using multiprocessing module

How do I create the python shared object of my class which can be modified by worker processes. I created the worker processes by using multiprocessing.Process module. I have some knowledge about multiprocessing.Manager module. Can any one explain with example how to register my class in to Manager, start the manager and create the shared object of my class.
Here is an example:
from multiprocessing import Process, Pool
from multiprocessing.managers import BaseManager
class MySharedClass(object):
stored_value = 0
def get(self):
return self.stored_value
def set(self, new_value):
self.stored_value = new_value
return self.stored_value
class MyManager(BaseManager):
pass
MyManager.register('MySharedClass', MySharedClass)
def worker ( proxy_object, i):
proxy_object.set( proxy_object.get() + i )
print ("id %d, sum %d" %(i, proxy_object.get()))
return proxy_object
if __name__ == '__main__':
manager = MyManager()
manager.start()
shared = manager.MySharedClass()
pool = Pool(5)
for i in range(33):
pool.apply(func=worker, args=(shared, i))
pool.close()
pool.join()
print "result: %d" % shared.get()
id 0, sum 0
id 1, sum 1
id 2, sum 3
...
id 31, sum 496
id 32, sum 528
result: 528
Another variant (have never use it in the real project):
from multiprocessing import Process, Pool
from multiprocessing.managers import BaseManager, NamespaceProxy
class MySharedClass(object):
def __init__(self):
self.stored_value = 0
def get(self):
return self.stored_value
def set(self, new_value):
self.stored_value = new_value
return self.stored_value
class MyManager(BaseManager):
pass
class MyProxy(NamespaceProxy):
_exposed_ = ('__getattribute__', '__setattr__', '__delattr__')# add 'get' to use get
#def get(self):
# callmethod = object.__getattribute__(self, '_callmethod')
# return callmethod('get')
MyManager.register('MySharedClass', MySharedClass, MyProxy)
def worker ( proxy_object, i):
proxy_object.stored_value = proxy_object.stored_value + i
print ("id %d, sum %d" %(i, proxy_object.stored_value))
return proxy_object
if __name__ == '__main__':
manager = MyManager()
manager.start()
shared = manager.MySharedClass()
print shared.stored_value
pool = Pool(5)
for i in range(33):
pool.apply(func=worker, args=(shared, i))
pool.close()
pool.join()
print "result: %d" % shared.stored_value

python multiprocessing process count

#coding:utf-8
import sys
import time
import os
import multiprocessing
class Worker(object):
def __init__(self):
self.progress = 0
self.task_info = None
def init(self):
pass
def status(self):
pass
def set_task_info(self, task_info):
self.task_info = task_info
def run(self, worker_status_meta_dict):
print multiprocessing.current_process()
print "process is %d" % self.progress
while self.progress < 5:
self.progress = self.progress +1
worker_status_meta_dict['state'] = 0
worker_status_meta_dict['status'] = "running"
time.sleep(2)
worker_status_meta_dict['state'] = 1
worker_status_meta_dict['status'] = "succeeded"
print "bavscan worker finished..."
if __name__ == "__main__":
worker = Worker()
worker_process_dict = multiprocessing.Manager().dict()
process = multiprocessing.Process(target=Worker.run, args=(worker, worker_process_dict))
process.start()
time.sleep(60)
This is a simple demo for python multiprocess.
The main process invoke the Worker.run method in a subprocess with multiprocessing.Process.
When run it in wondows 7, the main process will lauch two subprocess.
I find the problem in the "Python27\Lib\multiprocessing__init__.py"
def Manager():
'''
Returns a manager associated with a running server process
The managers methods such as `Lock()`, `Condition()` and `Queue()`
can be used to create shared objects.
'''
from multiprocessing.managers import SyncManager
m = SyncManager()
m.start()
return m
m.start() will lauch a subprocess to start the manager.

Python multiprocessing exiting cleanly

I've got a daemon that runs a number of child processes intended to maintain a telnet connection to collect data from a bunch of weather stations. I've set it up so that these child processes read from that telnet connection forever, passing the weather readings back to the parent process via a multiprocessing.Queue. I can't seem to get these child processes to exit cleanly when I stop the daemon with ./test.py stop. Is there an easy way to close the child processes on exit? A quick google mentioned someone using multiprocessing.Event, what's the best way to set this event on exit to ensure the processes exit? Here's our current code:
from daemon import runner
from multiprocessing import Process, Queue
import telnetlib
from django.utils.encoding import force_text
from observations.weather.models import WeatherStation
import os
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
def read_weather_data(name, ip_address, port, queue):
print "Started process to get data for", name
client = telnetlib.Telnet(ip_address, port)
while True:
response = client.read_until('\r\n'.encode('utf8'))
queue.put((name, force_text(response)))
client.close()
class App(object):
def __init__(self):
self.stdin_path = '/dev/null'
self.stdout_path = '/dev/tty'
self.stderr_path = '/dev/tty'
self.pidfile_path = '/tmp/process_weather.pid'
self.pidfile_timeout = 5
def run(self):
queue = Queue()
for station in WeatherStation.objects.filter(active=True):
p = Process(target=read_weather_data,
args=(station.name, station.ip_address, station.port,
queue,))
p.start()
while True:
name, data = queue.get()
print "Received data from ", name
print data
app = App()
daemon_runner = runner.DaemonRunner(app)
daemon_runner.do_action()
Seem to have found a way to do this, but am unsure about whether this is the best approach to take.
from daemon import runner
from multiprocessing import Process, Queue, Event
import telnetlib
from django.utils.encoding import force_text
from observations.weather.models import WeatherStation
import os
import signal
import errno
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
def read_weather_data(name, ip_address, port, queue, exit):
print "Started process to get data for", name
client = telnetlib.Telnet(ip_address, port)
while not exit.is_set():
response = client.read_until('\r\n'.encode('utf8'))
queue.put((name, force_text(response)))
print "exit called for", name
client.close()
def exit_handler(signum, frame):
print "exiting..."
class App(object):
def __init__(self):
self.stdin_path = '/dev/null'
self.stdout_path = '/dev/tty'
self.stderr_path = '/dev/tty'
self.pidfile_path = '/tmp/process_weather.pid'
self.pidfile_timeout = 5
def run(self):
exit = Event()
def exit_handler(signum, frame):
print "exiting..."
exit.set()
signal.signal(signal.SIGTERM, exit_handler)
queue = Queue()
workers = []
for station in WeatherStation.objects.filter(active=True):
p = Process(target=read_weather_data,
args=(station.name, station.ip_address, station.port,
queue, exit))
workers.append(p)
for worker in workers:
worker.start()
while True:
try:
name, data = queue.get()
except IOError as e:
# we received a signal whilst waiting for I/O
if e.errno != errno.EINTR:
raise
else:
break
print "Received data from ", name
print data
for worker in workers:
worker.join()
app = App()
daemon_runner = runner.DaemonRunner(app)
daemon_runner.do_action()

Categories

Resources