I am implementing a GUI project, and I want to use multiprocessing to accelerate the calculation. In order to tell the user that the calculation is finished, I use a threading to check whether the calculation is finished. My code is:
import sys
import threading
from PyQt5.QtWidgets import *
import multiprocessing
import time
class SubProcess(multiprocessing.Process):
def __init__(self, calFunc):
super().__init__()
self.calFunc = calFunc
self.recv, self.send = multiprocessing.Pipe(False)
def run(self) -> None:
self.calFunc()
self.send.send(True)
def IsFinished(self):
return self.recv.recv()
class SubThread(threading.Thread):
def __init__(self, calFunc, finishFunc):
super().__init__()
self.finishFunc = finishFunc
self.subProcess = SubProcess(calFunc)
def start(self) -> None:
super().start()
self.subProcess.start()
def run(self) -> None:
while self.subProcess.IsFinished() != True:
pass
self.finishFunc()
class Widget(QWidget):
def __init__(self):
super().__init__()
self.btn = QPushButton('calculation')
self.btn.clicked.connect(self.calBtnClick)
self.label = QLabel('not finish')
self.lay = QVBoxLayout()
self.setLayout(self.lay)
self.lay.addWidget(self.btn)
self.lay.addWidget(self.label)
def calBtnClick(self, check=False):
print('calculation')
self.subThread = SubThread(self.calculation, self.finish)
self.subThread.start()
def finish(self):
print('calculation finish')
self.label.setText('finish')
def calculation(self):
time.sleep(5)
if __name__ == '__main__':
app = QApplication(sys.argv)
win = Widget()
win.show()
app.exec()
When press the QPushButton, I want Widget.calculation run in the sub-process, and a sub-threading will check whether the sub-process is finished.
However, the code reports a bug:
Traceback (most recent call last):
File "C:/Users/zhq/Desktop/test/test2.py", line 58, in calBtnClick
self.subThread.start()
File "C:/Users/zhq/Desktop/test/test2.py", line 32, in start
self.subProcess.start()
File "C:\ProgramData\Anaconda3\envs\MedPro\lib\multiprocessing\process.py", line 112, in start
self._popen = self._Popen(self)
File "C:\ProgramData\Anaconda3\envs\MedPro\lib\multiprocessing\context.py", line 223, in _Popen
return _default_context.get_context().Process._Popen(process_obj)
File "C:\ProgramData\Anaconda3\envs\MedPro\lib\multiprocessing\context.py", line 322, in _Popen
return Popen(process_obj)
File "C:\ProgramData\Anaconda3\envs\MedPro\lib\multiprocessing\popen_spawn_win32.py", line 89, in __init__
reduction.dump(process_obj, to_child)
File "C:\ProgramData\Anaconda3\envs\MedPro\lib\multiprocessing\reduction.py", line 60, in dump
ForkingPickler(file, protocol).dump(obj)
TypeError: can't pickle Widget objects
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "C:\ProgramData\Anaconda3\envs\MedPro\lib\multiprocessing\spawn.py", line 105, in spawn_main
exitcode = _main(fd)
File "C:\ProgramData\Anaconda3\envs\MedPro\lib\multiprocessing\spawn.py", line 115, in _main
self = reduction.pickle.load(from_parent)
EOFError: Ran out of input
The bug is reported for self.subProcess.start().
How to fix this bug? Any suggestion is appreciated~~~
Update
Thank #Dipet, the code is modifed as:
import sys
import threading
from PyQt5.QtWidgets import *
import multiprocessing
import time
def calFunc():
time.sleep(5)
class SubProcess(multiprocessing.Process):
def __init__(self, calFunc):
super().__init__()
self.calFunc = calFunc
self.recv, self.send = multiprocessing.Pipe(False)
def run(self) -> None:
self.calFunc()
self.send.send(True)
def IsFinished(self):
return self.recv.recv()
class SubThread(threading.Thread):
def __init__(self, calFunc, finishFunc):
super().__init__()
self.finishFunc = finishFunc
self.subProcess = SubProcess(calFunc)
def start(self) -> None:
super().start()
self.subProcess.start()
def run(self) -> None:
while self.subProcess.IsFinished() != True:
pass
self.finishFunc()
class Widget(QWidget):
def __init__(self):
super().__init__()
self.btn = QPushButton('calculation')
self.btn.clicked.connect(self.calBtnClick)
self.label = QLabel('not finish')
self.lay = QVBoxLayout()
self.setLayout(self.lay)
self.lay.addWidget(self.btn)
self.lay.addWidget(self.label)
def calBtnClick(self, check=False):
print('calculation')
self.subThread = SubThread(calFunc, self.finish)
self.subThread.start()
def finish(self):
print('calculation finish')
self.label.setText('finish')
def calculation(self):
time.sleep(5)
if __name__ == '__main__':
app = QApplication(sys.argv)
win = Widget()
win.show()
app.exec()
Multiprocessing module provide args using pickle, so all values that you provided into subproces must support pickle protocol. You use self.finish as an argument for your worker, but this is a Widget method, but QWidget do not support pickling. And you see and error: TypeError: can't pickle Widget objects. Try to use staticmethod or common function.
Related
I want to run parallel processing, using the class but the code gives this error :
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "C:\Users\MonPc\AppData\Local\Programs\Python\Python37\lib\multiprocessing\spawn.py", line 105, in spawn_main
exitcode = _main(fd)
File "C:\Users\MonPc\AppData\Local\Programs\Python\Python37\lib\multiprocessing\spawn.py", line 115, in _main
self = reduction.pickle.load(from_parent)
EOFError: Ran out of input
This is just similar to the original code that I want to work on.
MyCode :
from multiprocessing import Process
class class1:
def __init__(self):
super().__init__()
txt = "Rachid"
p1 = class2(txt)
p1.start()
p1.join()
class class2(Process):
def __init__(self, txt):
Process.__init__(self)
self.txt = txt
def run (self):
print("*"*10)
print(self.txt)
print("*"*10)
class1()
How can I avoid getting this error ?
Quite new to multiprocessing here. I have a code that runs two processes. One to continuously receive data blocks from the server and put it inside a queue and the other to remove the data blocks from the queue and process it.
Below is my client code:
import socket
import turtle
import multiprocessing
from multiprocessing import Process, Queue
from tkinter import *
class GUI:
def __init__(self, master):
rec_data = recv_data()
self.master = master
master.title("Collision Detection")
self.input_label = Label(root, text="Input all the gratings set straight wavelength values in nm")
self.input_label.grid(row=0)
self.core_string = "Core "
self.entries = []
self.label_col_inc = 0
self.entry_col_inc = 1
self.core_range = range(1, 5)
for y in self.core_range:
self.core_text = self.core_string + str(y) + '_' + '25'
self.core_label = Label(root, text=self.core_text)
self.entry = Entry(root)
self.core_label.grid(row=1, column=self.label_col_inc, sticky=E)
self.entry.grid(row=1, column=self.entry_col_inc)
self.entries.append(self.entry)
self.label_col_inc += 2
self.entry_col_inc += 2
self.threshold_label = Label(root, text="Threshold in nm")
self.entry_threshold = Entry(root)
self.threshold_label.grid(row=2, sticky=E)
self.entry_threshold.grid(row=2, column=1)
self.light_label = Label(root, text='Status')
self.light_label.grid(row=3, column=3)
self.canvas = Canvas(root, width=150, height=50)
self.canvas.grid(row=4, column=3)
# Green light
self.green_light = turtle.RawTurtle(self.canvas)
self.green_light.shape('circle')
self.green_light.color('grey')
self.green_light.penup()
self.green_light.goto(0, 0)
# Red light
self.red_light = turtle.RawTurtle(self.canvas)
self.red_light.shape('circle')
self.red_light.color('grey')
self.red_light.penup()
self.red_light.goto(40, 0)
self.data_button = Button(root, text="Get data above threshold", command=rec_data.getData)
self.data_button.grid(row=5, column=0)
class recv_data:
def __init__(self):
self.buff_data = multiprocessing.Queue()
self.p1 = multiprocessing.Process(target=self.recvData)
self.p2 = multiprocessing.Process(target=self.calculate_threshold)
self.host = '127.0.0.1'
self.port = 5001
self.s = socket.socket()
self.s.connect((self.host, self.port))
# function to receive TCP data blocks
def getData(self):
len_message = self.s.recv(4)
bytes_length = int(len_message.decode('utf-8')) # for the self-made server
recvd_data = self.s.recv(bytes_length)
self.buff_data.put(recvd_data)
self.p1.start()
self.p2.start()
self.p1.join()
self.p2.join()
def recvData(self):
len_message = self.s.recv(4)
while len_message:
bytes_length = int(len_message.decode('utf-8')) # for the self-made server
recvd_data = self.s.recv(bytes_length)
self.buff_data.put(recvd_data)
len_message = self.s.recv(4)
else:
print('out of loop')
self.s.close()
def calculate_threshold(self):
rmv_data = self.buff_data.get()
stringdata = rmv_data.decode('utf-8')
rep_str = stringdata.replace(",", ".")
splitstr = rep_str.split()
# received wavelength values
inc = 34
wav_threshold = []
for y in gui.entries:
straight_wav = float(y.get())
wav = float(splitstr[inc])
wav_diff = wav - straight_wav
if wav_diff < 0:
wav_diff = wav_diff * (-1)
wav_threshold.append(wav_diff)
inc += 56
threshold = float(gui.entry_threshold.get())
for x in wav_threshold:
if (x > threshold):
gui.red_light.color('red')
gui.green_light.color('grey')
else:
gui.red_light.color('grey')
gui.green_light.color('green')
# function to write into the file
def write_file(self, data):
with open("Output.txt", "a") as text_file:
text_file.write('\t'.join(data[0:]))
text_file.write('\n')
if __name__ == '__main__':
root = Tk()
gui1 = GUI(root)
root.mainloop()
The error I get is shown below:
Exception in Tkinter callback
Traceback (most recent call last):
File "C:\Users\AppData\Local\Programs\Python\Python38-32\lib\tkinter\__init__.py", line 1883, in __call__
return self.func(*args)
File "C:/Users/PycharmProjects/GUI/GUI_multiprocess.py", line 85, in getData
self.p2.start()
File "C:\Users\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\process.py", line 121, in start
self._popen = self._Popen(self)
File "C:\Users\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\context.py", line 224, in _Popen
return _default_context.get_context().Process._Popen(process_obj)
File "C:\Users\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\context.py", line 326, in _Popen
return Popen(process_obj)
File "C:\Users\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\popen_spawn_win32.py", line 93, in __init__
reduction.dump(process_obj, to_child)
File "C:\Users\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\reduction.py", line 60, in dump
ForkingPickler(file, protocol).dump(obj)
TypeError: cannot pickle 'weakref' object
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "C:\Users\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\spawn.py", line 116, in spawn_main
exitcode = _main(fd, parent_sentinel)
File "C:\Users\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\spawn.py", line 126, in _main
self = reduction.pickle.load(from_parent)
EOFError: Ran out of input
What am I doing wrong here and how can I fix it? Any help is appreciated. Thank you!
I just came to the same traceback and managed to solve it. It was due to that an object had a running or exited Process as a variable and it was starting another Process using that object.
Problem
This is a minimal code to produce your error:
import multiprocessing
class Foo:
def __init__(self):
self.process_1 = multiprocessing.Process(target=self.do_stuff1)
self.process_2 = multiprocessing.Process(target=self.do_stuff2)
def do_multiprocessing(self):
self.process_1.start()
self.process_2.start()
def do_stuff1(self):
print("Doing 1")
def do_stuff2(self):
print("Doing 2")
if __name__ == '__main__':
foo = Foo()
foo.do_multiprocessing()
[out]:
Traceback (most recent call last):
File "myfile.py", line 21, in <module>
foo.do_multiprocessing()
File "myfile.py", line 11, in do_multiprocessing
self.process_2.start()
File "...\lib\multiprocessing\process.py", line 121, in start
self._popen = self._Popen(self)
File "...\lib\multiprocessing\context.py", line 224, in _Popen
return _default_context.get_context().Process._Popen(process_obj)
File "...\lib\multiprocessing\context.py", line 327, in _Popen
return Popen(process_obj)
File "...\lib\multiprocessing\popen_spawn_win32.py", line 93, in __init__
reduction.dump(process_obj, to_child)
File "...\lib\multiprocessing\reduction.py", line 60, in dump
ForkingPickler(file, protocol).dump(obj)
TypeError: cannot pickle 'weakref' object
Doing 1
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "...\lib\multiprocessing\spawn.py", line 116, in spawn_main
exitcode = _main(fd, parent_sentinel)
File "...\lib\multiprocessing\spawn.py", line 126, in _main
self = reduction.pickle.load(from_parent)
EOFError: Ran out of input
So the issue is that Foo contains also the running/exited process foo.process_1 when it starts foo.process_2.
Solution 1
Set foo.process_1 to None or something else. Or store the Processes somewhere else than in foo to prevent being passed when starting process_2.
...
def do_multiprocessing(self):
self.process_1.start()
self.process_1 = None # Remove exited process
self.process_2.start()
...
Solution 2
Remove the problematic variable (process_1) from pickling:
class Foo:
def __getstate__(self):
# capture what is normally pickled
state = self.__dict__.copy()
# remove unpicklable/problematic variables
state['process_1'] = None
return state
...
This seems to be problem in newer Python versions. My own code worked fine for 3.7 but failed due to this issue in 3.9.
I tested your code (from recv_data). Since you join the processes and need them, you should do the solution 2 or store the processes somewhere else than in recv_data. Not sure what other problems your code has.
When I was trying to use grid layout, it shows errors when I try to use more widgets but with one widget it shows no errors,I have reviewed the codes many times but i didn't find any solutions,and by the way I'm using python 3.7 and kivy version 2 Here's my code:
import kivy
from kivy.app import App
from kivy.uix.label import Label
from kivy.uix.gridlayout import GridLayout
from kivy.uix.textinput import TextInput
class My_Grid(GridLayout):
def __init__(self, **kwargs):
super(My_Grid, self).__init__(**kwargs)
self.cols = 4
self.rows = 3
self.add_widget(Label(text="First Name: "))
self.FirstName = TextInput(multiline=False)
self.add_widget(self.FirstName)
self.add_widget(Label(text="Last Name: "))
self.LastName = TextInput(mulyiline=False)
self.add_widget(self.LastName)
self.add_widget(Label(text="email: "))
self.Email = TextInput(mulyiline=False)
self.add_widget(self.Email)
class TestApp(App):
def build(self):
return My_Grid()
if __name__ == "__main__":
TestApp().run()
the errors when i use more widgets:
Traceback (most recent call last):
File "C:/Users/Asus/PycharmProjects/untitled3/my.py", line 29, in <module>
TestApp().run()
File "C:\Users\Asus\PycharmProjects\untitled3\venv\lib\site-packages\kivy\app.py", line 829, in run
root = self.build()
File "C:/Users/Asus/PycharmProjects/untitled3/my.py", line 25, in build
return My_Grid()
File "C:/Users/Asus/PycharmProjects/untitled3/my.py", line 18, in __init__
self.LastName = TextInput(mulyiline=False)
File "C:\Users\Asus\PycharmProjects\untitled3\venv\lib\site-packages\kivy\uix\textinput.py", line 527, in __init__
super(TextInput, self).__init__(**kwargs)
File "C:\Users\Asus\PycharmProjects\untitled3\venv\lib\site-packages\kivy\uix\behaviors\focus.py", line 367, in __init__
super(FocusBehavior, self).__init__(**kwargs)
File "C:\Users\Asus\PycharmProjects\untitled3\venv\lib\site-packages\kivy\uix\widget.py", line 350, in __init__
super(Widget, self).__init__(**kwargs)
File "kivy\_event.pyx", line 243, in kivy._event.EventDispatcher.__init__
TypeError: object.__init__() takes exactly one argument (the instance to initialize)
Process finished with exit code 1
At first, you have a mistake in the word 'multiline' in self.LastName and self.Email. Try to fix that.
I'm wrting a program that spawns a process and restarts the process on certain conditions. For example, if a child process doesn't send data anymore to the mother process, for a certain period of time, I want the mother process to terminate the child process and restart it. I thought I could use a thread to recieve data from a child process and restart the child process, but it doesn't work the way I thought.
import numpy as np
import multiprocessing as mp
import threading
import time
from apscheduler.schedulers.background import BackgroundScheduler
pipe_in, pipe_out = mp.Pipe()
class Mother():
def __init__(self):
self.pipe_out = pipe_out
self.proc = mp.Process(target = self.test_func, args=(pipe_in, ))
self.proc.start()
self.thread = threading.Thread(target=self.thread_reciever, args=(self.pipe_out, ))
self.thread.start()
def thread_reciever(self, pipe_out):
while True:
value = pipe_out.recv()
print(value)
if value == 5:
self.proc.terminate()
time.sleep(2)
self.proc = mp.Process(target = self.test_func)
self.proc.start()
def test_func(self, pipe_in):
for i in range(10):
pipe_in.send(i)
time.sleep(1)
if __name__ == '__main__':
r = Mother()
It prints out this error.
D:\>d:\python36-32\python.exe temp06.py
0
1
2
3
4
5
Exception in thread Thread-1:
Traceback (most recent call last):
File "d:\python36-32\lib\threading.py", line 916, in _bootstrap_inner
self.run()
File "d:\python36-32\lib\threading.py", line 864, in run
self._target(*self._args, **self._kwargs)
File "temp06.py", line 28, in thread_reciever
self.proc.start()
File "d:\python36-32\lib\multiprocessing\process.py", line 105, in start
self._popen = self._Popen(self)
File "d:\python36-32\lib\multiprocessing\context.py", line 223, in _Popen
return _default_context.get_context().Process._Popen(process_obj)
File "d:\python36-32\lib\multiprocessing\context.py", line 322, in _Popen
return Popen(process_obj)
File "d:\python36-32\lib\multiprocessing\popen_spawn_win32.py", line 65, in __init__
reduction.dump(process_obj, to_child)
File "d:\python36-32\lib\multiprocessing\reduction.py", line 60, in dump
ForkingPickler(file, protocol).dump(obj)
TypeError: can't pickle _thread.lock objects
D:\>Traceback (most recent call last):
File "<string>", line 1, in <module>
File "d:\python36-32\lib\multiprocessing\spawn.py", line 99, in spawn_main
new_handle = reduction.steal_handle(parent_pid, pipe_handle)
File "d:\python36-32\lib\multiprocessing\reduction.py", line 82, in steal_handle
_winapi.PROCESS_DUP_HANDLE, False, source_pid)
OSError: [WinError 87]
How could I start and terminate a process inside a thread? (I'm using a thread because it can synchronously recieve data from a different process) Or are there any other ways to do this job?
test_func as a global function
import numpy as np
import multiprocessing as mp
import threading
import time
from apscheduler.schedulers.background import BackgroundScheduler
pipe_in, pipe_out = mp.Pipe()
def test_func( pipe_in):
for i in range(10):
pipe_in.send(i)
time.sleep(1)
class Mother():
def __init__(self):
self.pipe_out = pipe_out
mp.freeze_support()
self.proc = mp.Process(target = test_func, args=(pipe_in, ))
self.proc.start()
self.thread = threading.Thread(target=self.thread_reciever, args=(self.pipe_out, ))
self.thread.start()
def thread_reciever(self, pipe_out):
while True:
value = pipe_out.recv()
print(value)
if value == 5:
self.proc.terminate()
time.sleep(2)
mp.freeze_support()
self.proc = mp.Process(target = test_func, args=(pipe_in,))
self.proc.start()
if __name__ == '__main__':
r = Mother()
OUTPUT
D:\> d:\python36-32\python.exe temp06.py
0
1
2
3
4
5
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "d:\python36-32\lib\multiprocessing\spawn.py", line 105, in spawn_main
exitcode = _main(fd)
File "d:\python36-32\lib\multiprocessing\spawn.py", line 115, in _main
self = reduction.pickle.load(from_parent)
AttributeError: Can't get attribute 'test_func' on <module '__main__' (built-in)>
under windows, as there is no fork syscall, python starts a new interpreter instance, use pickle/unpickle to reconstruct execution context, but thread.Lock is not picklable. while pickling self.test_func, self.thread reference to a thread.Lock object, makes it unpicklable.
you could simply change test_func to a plain global function, without thread object reference :
self.proc = mp.Process(target = test_func, args=(pipe_in,))
...
def test_func(pipe_in):
for i in range(10):
pipe_in.send(i)
time.sleep(1)
I am trying to start several processes in a class context which should share a queue:
import multiprocessing
import queue
class MyMulti:
def __init__(self):
self.myq = queue.Queue()
def printhello(self):
print("hello")
self.myq.put("hello")
def run(self):
for _ in range(5):
p = multiprocessing.Process(target=self.printhello)
p.start()
if __name__ == "__main__":
multiprocessing.freeze_support()
m = MyMulti()
m.run()
# at that point the queue is being filled in with five elements
This crashes with
C:\Python34\python.exe C:/Users/yop/dev/GetNessusScans/tests/testm.py
Traceback (most recent call last):
File "C:/Users/yop/dev/GetNessusScans/tests/testm.py", line 20, in <module>
m.run()
File "C:/Users/yop/dev/GetNessusScans/tests/testm.py", line 15, in run
p.start()
File "C:\Python34\lib\multiprocessing\process.py", line 105, in start
self._popen = self._Popen(self)
File "C:\Python34\lib\multiprocessing\context.py", line 212, in _Popen
return _default_context.get_context().Process._Popen(process_obj)
File "C:\Python34\lib\multiprocessing\context.py", line 313, in _Popen
return Popen(process_obj)
File "C:\Python34\lib\multiprocessing\popen_spawn_win32.py", line 66, in __init__
reduction.dump(process_obj, to_child)
File "C:\Python34\lib\multiprocessing\reduction.py", line 59, in dump
ForkingPickler(file, protocol).dump(obj)
_pickle.PicklingError: Can't pickle <class '_thread.lock'>: attribute lookup lock on _thread failed
An answer to a similar question suggested to have a worker uppermost function, which I adapted to my case as
import multiprocessing
import queue
def work(foo):
foo.printhello()
class MyMulti:
def __init__(self):
self.myq = queue.Queue()
def printhello(self):
print("hello")
self.myq.put("hello")
def run(self):
for _ in range(5):
p = multiprocessing.Process(target=work, args=(self,))
p.start()
if __name__ == "__main__":
multiprocessing.freeze_support()
m = MyMulti()
m.run()
# at that point the queue is being filled in with five elements
This crashes the same way, though.
Is there a way to start processes with methods as targets?
I should have used self.myq = multiprocessing.Queue() instead of queue.Queue().
multiprocessing.Queue() is, in addition of queue.Queue(), process safe.
I leave the question unanswered for now for someone to possibly comment if the whole approach is wrong.