I want to run parallel processing, using the class but the code gives this error :
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "C:\Users\MonPc\AppData\Local\Programs\Python\Python37\lib\multiprocessing\spawn.py", line 105, in spawn_main
exitcode = _main(fd)
File "C:\Users\MonPc\AppData\Local\Programs\Python\Python37\lib\multiprocessing\spawn.py", line 115, in _main
self = reduction.pickle.load(from_parent)
EOFError: Ran out of input
This is just similar to the original code that I want to work on.
MyCode :
from multiprocessing import Process
class class1:
def __init__(self):
super().__init__()
txt = "Rachid"
p1 = class2(txt)
p1.start()
p1.join()
class class2(Process):
def __init__(self, txt):
Process.__init__(self)
self.txt = txt
def run (self):
print("*"*10)
print(self.txt)
print("*"*10)
class1()
How can I avoid getting this error ?
Related
im running my function with multiprocessing implementation
def assign_task(self, module, command):
logging.debug("Assigning task for {0}".format(command._get_module_id()))
if self.queue is None:
self.queue = JoinableQueue()
if self.processes is None:
self.processes = [Process(target=self.do_execute) for i in range(self.max_process)]
for process in self.processes:
process.daemon = True
process.start()
logging.debug("Queuing message {0}".format(command._get_module_id()))
self.queue.put((module, command))
def do_execute(self):
# the code
but getting this error
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "C:\Users\muhammad.aqshol\AppData\Local\Programs\Python\Python38\lib\multiprocessing\spawn.py", line 107, in spawn_main
new_handle = reduction.duplicate(pipe_handle,
File "C:\Users\muhammad.aqshol\AppData\Local\Programs\Python\Python38\lib\multiprocessing\reduction.py", line 79, in duplicate
return _winapi.DuplicateHandle(
OSError: [WinError 6] The handle is invalid
Am i wrong on the implementation or something missing?
Quite new to multiprocessing here. I have a code that runs two processes. One to continuously receive data blocks from the server and put it inside a queue and the other to remove the data blocks from the queue and process it.
Below is my client code:
import socket
import turtle
import multiprocessing
from multiprocessing import Process, Queue
from tkinter import *
class GUI:
def __init__(self, master):
rec_data = recv_data()
self.master = master
master.title("Collision Detection")
self.input_label = Label(root, text="Input all the gratings set straight wavelength values in nm")
self.input_label.grid(row=0)
self.core_string = "Core "
self.entries = []
self.label_col_inc = 0
self.entry_col_inc = 1
self.core_range = range(1, 5)
for y in self.core_range:
self.core_text = self.core_string + str(y) + '_' + '25'
self.core_label = Label(root, text=self.core_text)
self.entry = Entry(root)
self.core_label.grid(row=1, column=self.label_col_inc, sticky=E)
self.entry.grid(row=1, column=self.entry_col_inc)
self.entries.append(self.entry)
self.label_col_inc += 2
self.entry_col_inc += 2
self.threshold_label = Label(root, text="Threshold in nm")
self.entry_threshold = Entry(root)
self.threshold_label.grid(row=2, sticky=E)
self.entry_threshold.grid(row=2, column=1)
self.light_label = Label(root, text='Status')
self.light_label.grid(row=3, column=3)
self.canvas = Canvas(root, width=150, height=50)
self.canvas.grid(row=4, column=3)
# Green light
self.green_light = turtle.RawTurtle(self.canvas)
self.green_light.shape('circle')
self.green_light.color('grey')
self.green_light.penup()
self.green_light.goto(0, 0)
# Red light
self.red_light = turtle.RawTurtle(self.canvas)
self.red_light.shape('circle')
self.red_light.color('grey')
self.red_light.penup()
self.red_light.goto(40, 0)
self.data_button = Button(root, text="Get data above threshold", command=rec_data.getData)
self.data_button.grid(row=5, column=0)
class recv_data:
def __init__(self):
self.buff_data = multiprocessing.Queue()
self.p1 = multiprocessing.Process(target=self.recvData)
self.p2 = multiprocessing.Process(target=self.calculate_threshold)
self.host = '127.0.0.1'
self.port = 5001
self.s = socket.socket()
self.s.connect((self.host, self.port))
# function to receive TCP data blocks
def getData(self):
len_message = self.s.recv(4)
bytes_length = int(len_message.decode('utf-8')) # for the self-made server
recvd_data = self.s.recv(bytes_length)
self.buff_data.put(recvd_data)
self.p1.start()
self.p2.start()
self.p1.join()
self.p2.join()
def recvData(self):
len_message = self.s.recv(4)
while len_message:
bytes_length = int(len_message.decode('utf-8')) # for the self-made server
recvd_data = self.s.recv(bytes_length)
self.buff_data.put(recvd_data)
len_message = self.s.recv(4)
else:
print('out of loop')
self.s.close()
def calculate_threshold(self):
rmv_data = self.buff_data.get()
stringdata = rmv_data.decode('utf-8')
rep_str = stringdata.replace(",", ".")
splitstr = rep_str.split()
# received wavelength values
inc = 34
wav_threshold = []
for y in gui.entries:
straight_wav = float(y.get())
wav = float(splitstr[inc])
wav_diff = wav - straight_wav
if wav_diff < 0:
wav_diff = wav_diff * (-1)
wav_threshold.append(wav_diff)
inc += 56
threshold = float(gui.entry_threshold.get())
for x in wav_threshold:
if (x > threshold):
gui.red_light.color('red')
gui.green_light.color('grey')
else:
gui.red_light.color('grey')
gui.green_light.color('green')
# function to write into the file
def write_file(self, data):
with open("Output.txt", "a") as text_file:
text_file.write('\t'.join(data[0:]))
text_file.write('\n')
if __name__ == '__main__':
root = Tk()
gui1 = GUI(root)
root.mainloop()
The error I get is shown below:
Exception in Tkinter callback
Traceback (most recent call last):
File "C:\Users\AppData\Local\Programs\Python\Python38-32\lib\tkinter\__init__.py", line 1883, in __call__
return self.func(*args)
File "C:/Users/PycharmProjects/GUI/GUI_multiprocess.py", line 85, in getData
self.p2.start()
File "C:\Users\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\process.py", line 121, in start
self._popen = self._Popen(self)
File "C:\Users\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\context.py", line 224, in _Popen
return _default_context.get_context().Process._Popen(process_obj)
File "C:\Users\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\context.py", line 326, in _Popen
return Popen(process_obj)
File "C:\Users\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\popen_spawn_win32.py", line 93, in __init__
reduction.dump(process_obj, to_child)
File "C:\Users\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\reduction.py", line 60, in dump
ForkingPickler(file, protocol).dump(obj)
TypeError: cannot pickle 'weakref' object
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "C:\Users\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\spawn.py", line 116, in spawn_main
exitcode = _main(fd, parent_sentinel)
File "C:\Users\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\spawn.py", line 126, in _main
self = reduction.pickle.load(from_parent)
EOFError: Ran out of input
What am I doing wrong here and how can I fix it? Any help is appreciated. Thank you!
I just came to the same traceback and managed to solve it. It was due to that an object had a running or exited Process as a variable and it was starting another Process using that object.
Problem
This is a minimal code to produce your error:
import multiprocessing
class Foo:
def __init__(self):
self.process_1 = multiprocessing.Process(target=self.do_stuff1)
self.process_2 = multiprocessing.Process(target=self.do_stuff2)
def do_multiprocessing(self):
self.process_1.start()
self.process_2.start()
def do_stuff1(self):
print("Doing 1")
def do_stuff2(self):
print("Doing 2")
if __name__ == '__main__':
foo = Foo()
foo.do_multiprocessing()
[out]:
Traceback (most recent call last):
File "myfile.py", line 21, in <module>
foo.do_multiprocessing()
File "myfile.py", line 11, in do_multiprocessing
self.process_2.start()
File "...\lib\multiprocessing\process.py", line 121, in start
self._popen = self._Popen(self)
File "...\lib\multiprocessing\context.py", line 224, in _Popen
return _default_context.get_context().Process._Popen(process_obj)
File "...\lib\multiprocessing\context.py", line 327, in _Popen
return Popen(process_obj)
File "...\lib\multiprocessing\popen_spawn_win32.py", line 93, in __init__
reduction.dump(process_obj, to_child)
File "...\lib\multiprocessing\reduction.py", line 60, in dump
ForkingPickler(file, protocol).dump(obj)
TypeError: cannot pickle 'weakref' object
Doing 1
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "...\lib\multiprocessing\spawn.py", line 116, in spawn_main
exitcode = _main(fd, parent_sentinel)
File "...\lib\multiprocessing\spawn.py", line 126, in _main
self = reduction.pickle.load(from_parent)
EOFError: Ran out of input
So the issue is that Foo contains also the running/exited process foo.process_1 when it starts foo.process_2.
Solution 1
Set foo.process_1 to None or something else. Or store the Processes somewhere else than in foo to prevent being passed when starting process_2.
...
def do_multiprocessing(self):
self.process_1.start()
self.process_1 = None # Remove exited process
self.process_2.start()
...
Solution 2
Remove the problematic variable (process_1) from pickling:
class Foo:
def __getstate__(self):
# capture what is normally pickled
state = self.__dict__.copy()
# remove unpicklable/problematic variables
state['process_1'] = None
return state
...
This seems to be problem in newer Python versions. My own code worked fine for 3.7 but failed due to this issue in 3.9.
I tested your code (from recv_data). Since you join the processes and need them, you should do the solution 2 or store the processes somewhere else than in recv_data. Not sure what other problems your code has.
I want to start a new process after main tkinter window is loaded.
I do it like This:
if __name__ == "__main__":
app = MainFrame()
print(("App loaded"))
canvasWindow = app.getCurrentTopFrame().start()
app.mainloop()
print("Window change")
after tkinter init i call function to start new thread
def start(self):
print("Before logic thread init")
lock = mp.Lock()
pro = mp.Process(target = self.manager.startTraining, args = (deley,lock))
pro.start()
This is startTrening fun
def startTraining(self,lock):
"""Starts training sequence"""
print("start tra")
for x in range(0,self.numOfGenerations):
print("x")
self.population.nextGeneration()
print("Iteration num ",x,"Fitness of best one is
",self.population.bestSalesman)
lock.acquire()
...........self.canvas.updateFrame(self.population.bestSalesman.dna.getAsListOfTuple())
lock.release()
This is updateFrame fun
def updateFrame(self,listOfPoints):
self.canvas.delete("all")
for y in listOfPoints:
self.canvas.create_oval(y[0], y[1], y[0]+5, y[1]+5, fill="Black")
li = cycle(listOfPoints)
p2 = next(li)
for x in listOfPoints:
p1,p2 = p2, next(li)
self.canvas.create_line(p1[0],p1[1],p2[0]+2,p2[1]+2)
if p2 == listOfPoints[-1]:
self.canvas.create_line(p2[0],p2[1],listOfPoints[0]+2,listOfPoints[0][1]+2)
self.canvas.pack()
I dont get why but behavior is such that main window does load and the
error occure
After init
After tkrasie
after start
App loaded
Before logic thread init
Traceback (most recent call last):
File "C:\Users\CrazyUrusai\git\WarsawSchoolOfAI\GeneticsAlgorithms\MainFrame.py", line 129, in <module>
canvasWindow = app.getCurrentTopFrame().start()
File "C:\Users\CrazyUrusai\git\WarsawSchoolOfAI\GeneticsAlgorithms\MainFrame.py", line 111, in start
pro.start()
File "C:\Users\CrazyUrusai\AppData\Local\Programs\Python\Python36\lib\multiprocessing\process.py", line 105, in start
self._popen = self._Popen(self)
File "C:\Users\CrazyUrusai\AppData\Local\Programs\Python\Python36\lib\multiprocessing\context.py", line 223, in _Popen
return _default_context.get_context().Process._Popen(process_obj)
File "C:\Users\CrazyUrusai\AppData\Local\Programs\Python\Python36\lib\multiprocessing\context.py", line 322, in _Popen
return Popen(process_obj)
File "C:\Users\CrazyUrusai\AppData\Local\Programs\Python\Python36\lib\multiprocessing\popen_spawn_win32.py", line 65, in __init__
reduction.dump(process_obj, to_child)
File "C:\Users\CrazyUrusai\AppData\Local\Programs\Python\Python36\lib\multiprocessing\reduction.py", line 60, in dump
ForkingPickler(file, protocol).dump(obj)
TypeError: can't pickle _tkinter.tkapp objects
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "C:\Users\CrazyUrusai\AppData\Local\Programs\Python\Python36\lib\multiprocessing\spawn.py", line 105, in spawn_main
exitcode = _main(fd)
File "C:\Users\CrazyUrusai\AppData\Local\Programs\Python\Python36\lib\multiprocessing\spawn.py", line 115, in _main
self = reduction.pickle.load(from_parent)
EOFError: Ran out of input
I'm wrting a program that spawns a process and restarts the process on certain conditions. For example, if a child process doesn't send data anymore to the mother process, for a certain period of time, I want the mother process to terminate the child process and restart it. I thought I could use a thread to recieve data from a child process and restart the child process, but it doesn't work the way I thought.
import numpy as np
import multiprocessing as mp
import threading
import time
from apscheduler.schedulers.background import BackgroundScheduler
pipe_in, pipe_out = mp.Pipe()
class Mother():
def __init__(self):
self.pipe_out = pipe_out
self.proc = mp.Process(target = self.test_func, args=(pipe_in, ))
self.proc.start()
self.thread = threading.Thread(target=self.thread_reciever, args=(self.pipe_out, ))
self.thread.start()
def thread_reciever(self, pipe_out):
while True:
value = pipe_out.recv()
print(value)
if value == 5:
self.proc.terminate()
time.sleep(2)
self.proc = mp.Process(target = self.test_func)
self.proc.start()
def test_func(self, pipe_in):
for i in range(10):
pipe_in.send(i)
time.sleep(1)
if __name__ == '__main__':
r = Mother()
It prints out this error.
D:\>d:\python36-32\python.exe temp06.py
0
1
2
3
4
5
Exception in thread Thread-1:
Traceback (most recent call last):
File "d:\python36-32\lib\threading.py", line 916, in _bootstrap_inner
self.run()
File "d:\python36-32\lib\threading.py", line 864, in run
self._target(*self._args, **self._kwargs)
File "temp06.py", line 28, in thread_reciever
self.proc.start()
File "d:\python36-32\lib\multiprocessing\process.py", line 105, in start
self._popen = self._Popen(self)
File "d:\python36-32\lib\multiprocessing\context.py", line 223, in _Popen
return _default_context.get_context().Process._Popen(process_obj)
File "d:\python36-32\lib\multiprocessing\context.py", line 322, in _Popen
return Popen(process_obj)
File "d:\python36-32\lib\multiprocessing\popen_spawn_win32.py", line 65, in __init__
reduction.dump(process_obj, to_child)
File "d:\python36-32\lib\multiprocessing\reduction.py", line 60, in dump
ForkingPickler(file, protocol).dump(obj)
TypeError: can't pickle _thread.lock objects
D:\>Traceback (most recent call last):
File "<string>", line 1, in <module>
File "d:\python36-32\lib\multiprocessing\spawn.py", line 99, in spawn_main
new_handle = reduction.steal_handle(parent_pid, pipe_handle)
File "d:\python36-32\lib\multiprocessing\reduction.py", line 82, in steal_handle
_winapi.PROCESS_DUP_HANDLE, False, source_pid)
OSError: [WinError 87]
How could I start and terminate a process inside a thread? (I'm using a thread because it can synchronously recieve data from a different process) Or are there any other ways to do this job?
test_func as a global function
import numpy as np
import multiprocessing as mp
import threading
import time
from apscheduler.schedulers.background import BackgroundScheduler
pipe_in, pipe_out = mp.Pipe()
def test_func( pipe_in):
for i in range(10):
pipe_in.send(i)
time.sleep(1)
class Mother():
def __init__(self):
self.pipe_out = pipe_out
mp.freeze_support()
self.proc = mp.Process(target = test_func, args=(pipe_in, ))
self.proc.start()
self.thread = threading.Thread(target=self.thread_reciever, args=(self.pipe_out, ))
self.thread.start()
def thread_reciever(self, pipe_out):
while True:
value = pipe_out.recv()
print(value)
if value == 5:
self.proc.terminate()
time.sleep(2)
mp.freeze_support()
self.proc = mp.Process(target = test_func, args=(pipe_in,))
self.proc.start()
if __name__ == '__main__':
r = Mother()
OUTPUT
D:\> d:\python36-32\python.exe temp06.py
0
1
2
3
4
5
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "d:\python36-32\lib\multiprocessing\spawn.py", line 105, in spawn_main
exitcode = _main(fd)
File "d:\python36-32\lib\multiprocessing\spawn.py", line 115, in _main
self = reduction.pickle.load(from_parent)
AttributeError: Can't get attribute 'test_func' on <module '__main__' (built-in)>
under windows, as there is no fork syscall, python starts a new interpreter instance, use pickle/unpickle to reconstruct execution context, but thread.Lock is not picklable. while pickling self.test_func, self.thread reference to a thread.Lock object, makes it unpicklable.
you could simply change test_func to a plain global function, without thread object reference :
self.proc = mp.Process(target = test_func, args=(pipe_in,))
...
def test_func(pipe_in):
for i in range(10):
pipe_in.send(i)
time.sleep(1)
Please see the code and the error in the last.
from brisa.core.reactors import install_default_reactor
reactor = install_default_reactor()
print reactor
import os
from brisa.upnp.device import Device, Service
from brisa.upnp.device.service import StateVariable
class BinaryLight(Device):
def __init__(self):
Device.__init__(self,
'urn:schemas-upnp-org:device:BinaryLight:1',
'Binary Light device')
class SwitchPower(Service):
def __init__(self):
Service.__init__(self,
'SwitchPower',
'urn:schemas-upnp-org:service:SwitchPower:1',
'',
os.getcwd() + '/SwitchPower-scpd.xml')
self.target = False
self.status = False
self.varin = StateVariable(self, "Status",
True, False, "boolean")
self.varin.subscribe_for_update(self.varUpdateCallback)
self.add_state_variable(self.varin)
def varUpdateCallback(self, name, value):
print name, 'was updated to', value
def SetTarget(self, *args, **kwargs):
self.target = kwargs['NewTargetValue']
self.status = self.target
self.set_state_variable('Status', self.target)
print 'Light switched ', {'1': 'on', '0': 'off'}.get(self.target, None)
return {}
def GetTarget(self, *args, **kwargs):
return {'RetTargetValue': self.target}
def soap_GetStatus(self, *args, **kwargs):
return {'ResultStatus': self.status}
if __name__ == '__main__':
device = BinaryLight()
device += SwitchPower()
# Start device
device.start()
# Setup main loop
reactor.add_after_stop_func(device.stop)
reactor.main()
I am getting an error:-
ankit#ubuntu:~/Desktop$ python binary_light.py Could you please tell me where I am doing mistake??
Error:
Traceback (most recent call last):
File "binary_light.py", line 8, in <module>
from brisa.upnp.device import Device, Service
File "/usr/local/lib/python2.6/dist-packages/brisa/upnp/device/__init__.py", line 8, in <module>
from brisa.upnp.device.device import Device
File "/usr/local/lib/python2.6/dist-packages/brisa/upnp/device/device.py", line 10, in <module>
from brisa.core import log, config, webserver, network
File "/usr/local/lib/python2.6/dist-packages/brisa/core/webserver.py", line 39, in <module>
raise RuntimeError('Network is down.')
RuntimeError: Network is down.
NEW ERROR:
Traceback (most recent call last):
File "binary_light.py", line 57, in <module>
device = BinaryLight()
File "binary_light.py", line 21, in __init__
'Binary Light device')
File "/usr/local/lib/python2.6/dist-packages/brisa/upnp/device/device.py", line 83, in __init__
additional_headers=additional_ssdp_headers)
File "/usr/local/lib/python2.6/dist-packages/brisa/upnp/ssdp.py", line 71, in __init__
data_callback=self._datagram_received)
File "/usr/local/lib/python2.6/dist-packages/brisa/core/network_listeners.py", line 188, in __init__
self._create_socket(shared_socket)
File "/usr/local/lib/python2.6/dist-packages/brisa/core/network_listeners.py", line 227, in _create_socket
"Couldn't bind address")
brisa.core.network_listeners.CannotListenError
please let me know where I am doing mistake??
As others have mentioned, that's not an error. However, my guess would be that you have an error and something's going wrong somewhere or you wouldn't have bothered posting here. Try removing the print statement. Then let us know if you get any more "errors".
This is not an error. On line 3 you are printing the object "reactor".
(In response to updated question)
Well, the error suggests it can't find the network.
Looking at the code, trying doing this at the start of your script:
import brisa
brisa.__enable_offline_mode__ = True
from brisa.core.reactors import install_default_reactor