I have the following code in two modules.
Module "main":
#!/usr/bin/python
import threading
import b
a = 0
def api_a():
global a
print("api_a()")
a = 1
def main():
global a
thread_B = b.B_thread()
print("a = " + str(a))
thread_B.start()
# api_a()
thread_B.join()
print("a = " + str(a))
if __name__ == '__main__':
main()
Module "B":
#!/usr/bin/python
import threading
import main
class B_thread (threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def run(self):
print("Starting " + self.name)
B_process()
print("Exiting " + self.name)
def B_process():
main.api_a()
If I run this code I get:
a = 0
Starting Thread-1
api_a()
Exiting Thread-1
a = 0
Why is variable "a" not set to "1" ?
If I activate the commented line of code in Module "main" ( api_a() ) the variable "a" will be set to "1". Why is the variable not set when the function api_a() is called via the thread?
In the example code I skipped the use of locks to make the code thread-safe.
Any ideas?
Thanks a lot in advance,
Thomas
Such behavior occurs, because you passed main.py as the argument to python (python main.py).
When you execute a script in such way, python interprets main module as __main__, but b module still updates main.a variable instead of __main__.a, because b has re-imported your main.py module as main.
So to make it work, we can either update __main__.a in api_a function from perspective of b module OR print main.a variable instead of __main__.a from perspective of __main__ module.
def api_a():
print("api_a()")
import sys
sys.modules['__main__'].a = 1
or
def main():
...
thread_B.join()
import sys
print("a = " + str(sys.modules['main'].a))
__main__ docs
Related
I am trying to share a string between processes with multiprocessing.Value(c_wchar_p, ""). Program exits unexpectedly without exception with code (0xC0000005). I am guessing that problem is somehow caused by main process trying to access memory of child process. Issue occurs only when ctypes c_char_p/c_wchar_p are used. Thread works fine. Is this expected beheavor?
Steps to reproduce:
import multiprocessing
import threading
import ctypes
import time
class ProcessExample:
def __init__(self, proc: bool):
self.shared_bool = multiprocessing.Value(ctypes.c_bool, False)
self.shared_string = multiprocessing.Value(ctypes.c_wchar_p, "one")
self.shared_byte = multiprocessing.Value(ctypes.c_char_p, b"one")
if proc:
self.th_proc = multiprocessing.Process(target=self.target, args=(self.shared_bool, self.shared_string, self.shared_byte))
else:
self.th_proc = threading.Thread(target=self.target, args=(self.shared_bool, self.shared_string, self.shared_byte))
self.th_proc.start()
#staticmethod
def target(shared_bool, shared_string, shared_byte):
shared_bool.value = True
shared_string.value = "two"
shared_byte.value = b"two"
def print_values(self):
self.th_proc.join()
print(self.shared_bool.value)
print(self.shared_string.value) # Exits
print(self.shared_byte.value) # Exits
if __name__ == "__main__":
# example = ProcessExample(False) # Works
example = ProcessExample(True)
time.sleep(1)
example.print_values()
Output
True
Process finished with exit code -1073741819 (0xC0000005)
I think the problem is in multiprocessing.Value(ctypes.c_wchar_p, "one") and multiprocessing.Value(ctypes.c_char_p, b"one"). To have string/byte string initialized correctly, use multiprocessing.Array:
import multiprocessing
import threading
import ctypes
import time
class ProcessExample:
def __init__(self, proc: bool):
self.shared_bool = multiprocessing.Value(ctypes.c_bool, False)
self.shared_string = multiprocessing.Array(ctypes.c_wchar, "one")
self.shared_byte = multiprocessing.Array(ctypes.c_char, b"one")
if proc:
self.th_proc = multiprocessing.Process(
target=self.target,
args=(self.shared_bool, self.shared_string, self.shared_byte),
)
else:
self.th_proc = threading.Thread(
target=self.target,
args=(self.shared_bool, self.shared_string, self.shared_byte),
)
self.th_proc.start()
#staticmethod
def target(shared_bool, shared_string, shared_byte):
shared_bool.value = True
shared_string[:] = "two"
shared_byte[:] = b"two"
def print_values(self):
self.th_proc.join()
print(self.shared_bool.value)
print(self.shared_string[:])
print(self.shared_byte[:])
if __name__ == "__main__":
example = ProcessExample(True)
time.sleep(1)
example.print_values()
Prints:
True
two
b'two'
I am trying to make use of a global variable which is updated in func1() and then used in func2().
When I call func2() simply without multiprocessing, it utilizes the updated value of the global variable. However, when func2() is invoked using multiprocessing it uses the old value of the global variable.
How can I make use of the updated value of the global variable while multiprocessing?
Given below is the simple code to demonstrate this:
import multiprocessing
from multiprocessing import Process
myGlobal = None
def func1():
global myGlobal
myGlobal = 42
def func2():
print(myGlobal)
if __name__ == "__main__":
print("before update")
func2()
func1()
print("after update")
func2()
print("Using multiprocessing")
list_process = []
for i in range(2):
p = Process(target=func2)
list_process.append(p)
p.start()
for i in range(2):
list_process[i].join()
Output on console:
before update
None
after update
42
Using multiprocessing
None
None
Process finished with exit code 0
I'm trying to pass a dict with input_data from Module1.py to a script that is run with subprocess and os.sys in Python 3 in Module2.py but it doesn't work. I'm first passing the dict, input_data, to main() in Module3.py and there to Module1.py, which returns the processed dict from the main function in Module1.py, but when I then run the script, if name == 'main':, with os.system it doesn't get the dict. That is, how can I pass a dict -object through main in Module3 where I first process it in Module1.py,return the processed dict and then use it as input when running the script in Module2.py through main in Module3.py ?
# Module 1
def main(input_data):
... code...
returns dict_with_processed_input_data
# Module 2
class Parser(threading.Thread):
..code..
if __name__ == '__main__':
# input data from Module 1 should be
# imported and used here, function calls for
# workers that take the input data from
# Module 1
# Module 3
def main(input_data):
#takes input_data and passes it to Module 1
input_data = Module1.main(input_data )
#runs the script in Module2 with os.sys
import os
os.system('python3 Module2.py')
The input_data from Module1.py is not passed to the
script in subprocess
#calls on function that reads the script
#in Module2.py with subprocess
import subprocess
def get_script_data():
tmp = None
proc =subprocess.Popen(['python3','Module2.py'],stdout=subprocess.PIPE)
while True:
line = proc.stdout.readline()
if bool(line) == False:
break
line = json.dumps(line.decode("utf-8"))
tmp = line.rstrip()
return tmp
res = get_script_data()
I've tried with writing a main function in Module2.py and return it to the script but it also #doesn't work with the script and subprocees.
# Module 2
def main(input_data):
return input_data
if __name__ == '__main__':
input_data = main()
The if name == 'main': in Module2 has worker calls for functions through a threading.Thread class
class Parser(threading.Thread):
""" A threading class that executes tasks from a task queue"""
def __init__(self,threads):
threading.Thread.__init__(self)
super(Parser, self).__init__()
self.tasks = Queue(threads) # queue that takes the number of threads that should be generated
for _ in range(threads): self.tasks # generates threads for events in self.tasks
self.daemon = True
self.work_begin = Queue() # worker queue
self.work_queue = PriorityQueue() # worker queue
def worker1(self, *args, **kwargs):
... code..
if __name__ == '__main__':
pool = Parser(11)
# the dict input_data needs to be imported, or read as a script here
# to be sent through worker1
pool.repeat_task(pool.n_repeats,pool.worker1, input_data)
Could you please explain me these basics.
server_init() -> start_db() -> it makes variable state = 5 -> server starts start_client() in 2 seconds -> client calls get_state() and gets value = 5
Is it right that client sees 5? How long will module variables live?
Thanks.
server.py:
def t3():
time.sleep(2)
start_client()
def server_init():
start_db()
thread = threading.Thread(target=t3)
thread.start()
server_init()
db.py
state = 0
q = queue.Queue()
def t2():
global state
while state < 5:
state += 1
global q
q.put(state)
def start_db():
thread = threading.Thread(target=t2)
thread.start()
def get_state():
print('q.get(): %s' % q.get())
print('state: %s' % state)
client.py
def t1():
get_state() # prints 5
def start_client():
print('start_client')
thread = threading.Thread(target=t1)
thread.start()
Modules are stored in the global mapping sys.modules. As long as there is any reference available to a module object, it'll remain alive. Typically, modules are not removed from sys.modules until the Python interpreter shuts down.
I'm trying to run a function function of a class Foo in multiple subprocesses created by multiprocessing.pool in my main fileMain.py.
import sys
import multiprocessing
from Foo import Foo
f = Foo()
def get(n):
print f.ID
sys.stdout.flush() # Print the used instance's ID immediately
return f.function(n)
def calculate():
s = pool.map_async(get,range(4))
return s.get()
if __name__ == '__main__':
pool = multiprocessing.Pool(processes=4)
result = calculate()
pool.close()
The class Foois defined in Foo.py like
import random
class Foo():
def __init__(self):
print 'Initialized!'
self.ID = random.random() # Assign an unique ID to spot instance
pass
def function(self,x):
return x**2
The output I get is (with the ID obviously randomly)
Initialized!
Initialized!
0.955181146828
0.955181146828
0.955181146828
0.955181146828
>>> Initialized!
Initialized!
Initialized!
I want to avoid that a new instance of Foo is created by every subprocess. Why is this happening although all subprocesses then use the same instance as desired?
Notice also that f = Foo() cannot be placed after if __name__ == '__main__': or else
NameError: global name 'f' is not defined