Develop Reference

TypeError: cannot pickle 'weakref' object

Quite new to multiprocessing here. I have a code that runs two processes. One to continuously receive data blocks from the server and put it inside a queue and the other to remove the data blocks from the queue and process it.
Below is my client code:
import socket
import turtle
import multiprocessing
from multiprocessing import Process, Queue
from tkinter import *
class GUI:
def __init__(self, master):
rec_data = recv_data()
self.master = master
master.title("Collision Detection")
self.input_label = Label(root, text="Input all the gratings set straight wavelength values in nm")
self.input_label.grid(row=0)
self.core_string = "Core "
self.entries = []
self.label_col_inc = 0
self.entry_col_inc = 1
self.core_range = range(1, 5)
for y in self.core_range:
self.core_text = self.core_string + str(y) + '_' + '25'
self.core_label = Label(root, text=self.core_text)
self.entry = Entry(root)
self.core_label.grid(row=1, column=self.label_col_inc, sticky=E)
self.entry.grid(row=1, column=self.entry_col_inc)
self.entries.append(self.entry)
self.label_col_inc += 2
self.entry_col_inc += 2
self.threshold_label = Label(root, text="Threshold in nm")
self.entry_threshold = Entry(root)
self.threshold_label.grid(row=2, sticky=E)
self.entry_threshold.grid(row=2, column=1)
self.light_label = Label(root, text='Status')
self.light_label.grid(row=3, column=3)
self.canvas = Canvas(root, width=150, height=50)
self.canvas.grid(row=4, column=3)
# Green light
self.green_light = turtle.RawTurtle(self.canvas)
self.green_light.shape('circle')
self.green_light.color('grey')
self.green_light.penup()
self.green_light.goto(0, 0)
# Red light
self.red_light = turtle.RawTurtle(self.canvas)
self.red_light.shape('circle')
self.red_light.color('grey')
self.red_light.penup()
self.red_light.goto(40, 0)
self.data_button = Button(root, text="Get data above threshold", command=rec_data.getData)
self.data_button.grid(row=5, column=0)
class recv_data:
def __init__(self):
self.buff_data = multiprocessing.Queue()
self.p1 = multiprocessing.Process(target=self.recvData)
self.p2 = multiprocessing.Process(target=self.calculate_threshold)
self.host = '127.0.0.1'
self.port = 5001
self.s = socket.socket()
self.s.connect((self.host, self.port))
# function to receive TCP data blocks
def getData(self):
len_message = self.s.recv(4)
bytes_length = int(len_message.decode('utf-8')) # for the self-made server
recvd_data = self.s.recv(bytes_length)
self.buff_data.put(recvd_data)
self.p1.start()
self.p2.start()
self.p1.join()
self.p2.join()
def recvData(self):
len_message = self.s.recv(4)
while len_message:
bytes_length = int(len_message.decode('utf-8')) # for the self-made server
recvd_data = self.s.recv(bytes_length)
self.buff_data.put(recvd_data)
len_message = self.s.recv(4)
else:
print('out of loop')
self.s.close()
def calculate_threshold(self):
rmv_data = self.buff_data.get()
stringdata = rmv_data.decode('utf-8')
rep_str = stringdata.replace(",", ".")
splitstr = rep_str.split()
# received wavelength values
inc = 34
wav_threshold = []
for y in gui.entries:
straight_wav = float(y.get())
wav = float(splitstr[inc])
wav_diff = wav - straight_wav
if wav_diff < 0:
wav_diff = wav_diff * (-1)
wav_threshold.append(wav_diff)
inc += 56
threshold = float(gui.entry_threshold.get())
for x in wav_threshold:
if (x > threshold):
gui.red_light.color('red')
gui.green_light.color('grey')
else:
gui.red_light.color('grey')
gui.green_light.color('green')
# function to write into the file
def write_file(self, data):
with open("Output.txt", "a") as text_file:
text_file.write('\t'.join(data[0:]))
text_file.write('\n')
if __name__ == '__main__':
root = Tk()
gui1 = GUI(root)
root.mainloop()
The error I get is shown below:
Exception in Tkinter callback
Traceback (most recent call last):
File "C:\Users\AppData\Local\Programs\Python\Python38-32\lib\tkinter\__init__.py", line 1883, in __call__
return self.func(*args)
File "C:/Users/PycharmProjects/GUI/GUI_multiprocess.py", line 85, in getData
self.p2.start()
File "C:\Users\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\process.py", line 121, in start
self._popen = self._Popen(self)
File "C:\Users\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\context.py", line 224, in _Popen
return _default_context.get_context().Process._Popen(process_obj)
File "C:\Users\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\context.py", line 326, in _Popen
return Popen(process_obj)
File "C:\Users\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\popen_spawn_win32.py", line 93, in __init__
reduction.dump(process_obj, to_child)
File "C:\Users\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\reduction.py", line 60, in dump
ForkingPickler(file, protocol).dump(obj)
TypeError: cannot pickle 'weakref' object
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "C:\Users\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\spawn.py", line 116, in spawn_main
exitcode = _main(fd, parent_sentinel)
File "C:\Users\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\spawn.py", line 126, in _main
self = reduction.pickle.load(from_parent)
EOFError: Ran out of input
What am I doing wrong here and how can I fix it? Any help is appreciated. Thank you!

I just came to the same traceback and managed to solve it. It was due to that an object had a running or exited Process as a variable and it was starting another Process using that object.
Problem
This is a minimal code to produce your error:
import multiprocessing
class Foo:
def __init__(self):
self.process_1 = multiprocessing.Process(target=self.do_stuff1)
self.process_2 = multiprocessing.Process(target=self.do_stuff2)
def do_multiprocessing(self):
self.process_1.start()
self.process_2.start()
def do_stuff1(self):
print("Doing 1")
def do_stuff2(self):
print("Doing 2")
if __name__ == '__main__':
foo = Foo()
foo.do_multiprocessing()
[out]:
Traceback (most recent call last):
File "myfile.py", line 21, in <module>
foo.do_multiprocessing()
File "myfile.py", line 11, in do_multiprocessing
self.process_2.start()
File "...\lib\multiprocessing\process.py", line 121, in start
self._popen = self._Popen(self)
File "...\lib\multiprocessing\context.py", line 224, in _Popen
return _default_context.get_context().Process._Popen(process_obj)
File "...\lib\multiprocessing\context.py", line 327, in _Popen
return Popen(process_obj)
File "...\lib\multiprocessing\popen_spawn_win32.py", line 93, in __init__
reduction.dump(process_obj, to_child)
File "...\lib\multiprocessing\reduction.py", line 60, in dump
ForkingPickler(file, protocol).dump(obj)
TypeError: cannot pickle 'weakref' object
Doing 1
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "...\lib\multiprocessing\spawn.py", line 116, in spawn_main
exitcode = _main(fd, parent_sentinel)
File "...\lib\multiprocessing\spawn.py", line 126, in _main
self = reduction.pickle.load(from_parent)
EOFError: Ran out of input
So the issue is that Foo contains also the running/exited process foo.process_1 when it starts foo.process_2.
Solution 1
Set foo.process_1 to None or something else. Or store the Processes somewhere else than in foo to prevent being passed when starting process_2.
...
def do_multiprocessing(self):
self.process_1.start()
self.process_1 = None # Remove exited process
self.process_2.start()
...
Solution 2
Remove the problematic variable (process_1) from pickling:
class Foo:
def __getstate__(self):
# capture what is normally pickled
state = self.__dict__.copy()
# remove unpicklable/problematic variables
state['process_1'] = None
return state
...
This seems to be problem in newer Python versions. My own code worked fine for 3.7 but failed due to this issue in 3.9.
I tested your code (from recv_data). Since you join the processes and need them, you should do the solution 2 or store the processes somewhere else than in recv_data. Not sure what other problems your code has.

Paho connection problem - Mosquitto: A network protocol error occurred when communicating with the broker

I run mosquitto (docker image - on external server)
After some period of time, my mosquitto does not work and show me this error (I do not know why)
A network protocol error occurred when communicating with the broker.
When this mosquitto error occures and I try connect/publish, (python) paho.mqtt throws me this:
Traceback (most recent call last):
File "/usr/lib/python3.5/threading.py", line 914, in _bootstrap_inner
self.run()
File "/usr/lib/python3.5/threading.py", line 862, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.5/dist-packages/paho/mqtt/client.py", line
2913, in _thread_main
self.loop_forever(retry_first_connection=True)
File "/usr/local/lib/python3.5/dist-packages/paho/mqtt/client.py", line
1578, in loop_forever
rc = self.loop(timeout, max_packets)
File "/usr/local/lib/python3.5/dist-packages/paho/mqtt/client.py", line
1072, in loop
rc = self.loop_read(max_packets)
File "/usr/local/lib/python3.5/dist-packages/paho/mqtt/client.py", line
1374, in loop_read
rc = self._packet_read()
File "/usr/local/lib/python3.5/dist-packages/paho/mqtt/client.py", line
2071, in _packet_read
rc = self._packet_handle()
File "/usr/local/lib/python3.5/dist-packages/paho/mqtt/client.py", line
2560, in _packet_handle
return self._handle_publish()
File "/usr/local/lib/python3.5/dist-packages/paho/mqtt/client.py", line
2728, in _handle_publish
(topic, packet) = struct.unpack(pack_format, packet) struct.error: bad char in struct format
It looks like my paho.mqtt.client is still connected but even if I restart mosquitto and I use mosquitto_sub (mosquitto error it's gone) I still do not see messages from my program. I need to restart program but it is not satisfying solution.
Could you tell me please, why mosquitto shows me this error and/or how can I catch (python) paho.mqtt socket.error (It's possible?)
I think that this part of code should be enought :)
mqtt_client.py
class MQttClient:
client = None
connected_flag = False
loop_is_working_flag = False
broker_is_online_flag = True
subscribe_command_channel_flag = True
mosquitto_ip = None
def __init__(self, config):
self.config = config
self.logger = Logger(config, self)
def connect(self, ip_address='127.0.0.1', port=1883, keepalive=4):
self.mosquitto_ip = ip_address
self.client = mqtt.Client(self.config.get(REQUIRE_INI_SECTION, DEVICE_ID_INI_PARAMETER), clean_session=False)
self.client.on_connect = self.on_connect
self.client.on_disconnect = self.on_disconnect
try:
self.client.connect(ip_address, port, keepalive=keepalive)
self.client.loop_start()
self.wait_for_connect()
self.loop_is_working_flag = True
except (OSError, ConnectionRefusedError):
self.logger.error(CAN_NOT_CONNECT_WITH_IP_ADDRESS_ERROR.format(ip_address), send_to_mosquitto=False)
def publish_message(self, message, topic, retain=False):
return self.client.publish(self.config.get(REQUIRE_INI_SECTION, BUILDING_NAME_INI_PARAMETER)
+ '/' + topic + '/' +
self.config.get(REQUIRE_INI_SECTION, DEVICE_ID_INI_PARAMETER), message, qos=1,
retain=retain)
def on_connect(self, client, userdata, flags, rc):
if rc is 0:
self.connected_flag = True
self.logger.info(CONNECTED_TO_THE_MOSQUITO_INFO.format(self.mosquitto_ip), send_to_mosquitto=True)
def on_disconnect(self, client, userdata, rc):
if rc is not 0:
self.connected_flag = False
self.logger.error(DISCONNECTED_FROM_THE_MOSQUITO_ERROR.format(
self.mosquitto_ip, str(rc)), send_to_mosquitto=True)
def wait_for_connect(self):
while not self.connected_flag:
pass
...
And usage:
controller.py
class Controller:
def __init__(self):
self.config = configparser.ConfigParser()
self.config.read(CONFIG_FILE_PATH)
self.client = MQttClient(self.config)
self.__connect_to_mosquitto()
def __connect_to_mosquitto(self):
if Network.host_is_available(self.config.get(NETWORK_INI_SECTION, BROKER_IP_INI_PARAMETER)):
if not self.client.loop_is_working_flag:
self.client.connect(ip_address=self.config.get(NETWORK_INI_SECTION, BROKER_IP_INI_PARAMETER))
time.sleep(0.2)
self.client.publish_message(self.json_parser.get_json_welcome_data(), MOSQUITTO_TOPIC_WELCOME_DATA)
else:
self.logger.error(NO_CONNECTION_WITH_BROKER_ERROR, send_to_mosquitto=False)
Network.restart_network_interface_if_there_is_no_connection(
self.config.get(NETWORK_INI_SECTION, ROUTER_IP_INI_PARAMETER))
...
def execute(self):
while self.client.connected_flag and self.client.broker_is_online_flag:
...
if self.client.publish_message(data, MOSQUITTO_TOPIC_DATA)[0] is not 0:
...
self.logger.error(CAN_NOT_SEND_DATA_TO_MOSQUITTO_ERROR)
break
I checked and this method returns 0 (success). But I do not get messages on mosquitto_sub
self.client.publish_message(data, MOSQUITTO_TOPIC_DATA)[0]

Updating Python tkinter label in realtime

I want to update my label in python after the loop has started.
Sadly I get the error
File "mygui.py", line 19, in <module>
GUI.user_said("General Kenobi")
File "mygui.py", line 16, in user_said
self.my_label['text'] = self.label_text
File "/usr/local/Cellar/python3/3.6.4_2/Frameworks/Python.framework/Versions/3.6/lib/python3.6/tkinter/__init__.py", line 1486, in __setitem__
self.configure({key: value})
File "/usr/local/Cellar/python3/3.6.4_2/Frameworks/Python.framework/Versions/3.6/lib/python3.6/tkinter/__init__.py", line 1479, in configure
return self._configure('configure', cnf, kw)
File "/usr/local/Cellar/python3/3.6.4_2/Frameworks/Python.framework/Versions/3.6/lib/python3.6/tkinter/__init__.py", line 1470, in _configure
self.tk.call(_flatten((self._w, cmd)) + self._options(cnf))
_tkinter.TclError: invalid command name ".!label"
which doesn't help me at all...
My Code:
from tkinter import *
class MyGUI:
"""A simple class"""
def __init__(self):
self.__main_window = Tk()
self.label_text = "Hello There"
self.my_label = Label(self.__main_window, text=self.label_text)
self.my_label.pack()
self.__main_window.mainloop()
def user_said(self, users_request):
"""Returns what the user says"""
self.label_text = "You said:\n{}\n\n".format(users_request)
self.my_label.config(text=self.label_text)
GUI = MyGUI()
GUI.user_said("General Kenobi")
I'm very glad if someone helps me to find a solution for my problem.

Because MyGUI.__init__ calls self.__main_window.mainloop(), it will not return until the main window is destroyed. Therefore, by the time you call GUI.user_said("General Kenobi"), none of the widgets exist any more.

exception raised by tornado

I wrote a simple http server with Tornado, but sometimes it raise exception below, self.session.response runs in a threadpool.
code of server:
class AuToServer(object):
module_path = os.path.split(os.path.realpath(__file__))[0] + '/../module/'
def __init__(self, prefix=DEFAULT_PREFIX, work_root=module_path, module=[], **kwargs):
self._prefix = prefix
self._thread_pool = ExecutorDelegate(20)
self._module_manager = ModuleManager(work_root)
self.initilize(module, kwargs)
def initilize(self, modules, kwargs):
self._module_manager.set_args(kwargs)
if not self._module_manager.prepare(modules):
print "initilize module fail. [%s]" % (str(modules))
raise AppInitError("initilize %s fail" % (modules))
def __call__(self, request): # request handler
session = Session(request)
if not session.parse(self._prefix):
print "parse query fail. [%s]" % (session.query)
session.response("url parse fail [%s]\n" % session.query)
return
self._thread_pool.submit(WorkerHander(self._module_manager, session))
def start(self, port=8880):
http_server = tornado.httpserver.HTTPServer(self, max_header_size=128*1024)
http_server.listen(port)
tornado.ioloop.IOLoop.instance().start()
code of WorkerHander:
class WorkerHander(BaseExecutor):
def __init__(self, module_manage, session):
self.module_manage = module_manage
self.session = session
super(WorkerHander, self).__init__(self)
def run(self):
method = self.session.get_command()
if not method:
self.session.response("invalid url [%s]\n" % (self.session.query))
return
context = self.module_manage.run(method, self.session)
try:
if context:
self.session.response(context) # raise exception
else:
self.session.response("None\n")
except Exception, error:
Logger.warning("session response fail. [%s][%s]" % (self.session.query, traceback.format_exc()))
code of session.response
def response(self, context):
if not self.request:
return False
if isinstance(context, unicode):
context = context.encode("utf8")
self.request.write(utf8("%s %d\r\n\r\n%s" % (
HTTP_HEADER,
len(context),
context)))
Logger.debug("QUERY:[cmd:%s][%s] [%s] [%s]" % (
self.command,
time.time() - self.begin_time,
self.request.remote_ip,
self.query))
self.request.finish()
return True
exception:
2016-02-29 16:29:26,852 WARNING: server.py:run:94: session response fail. [/auto?start_timestamp=1456730772&_cmd=get_index_info][Traceback (most recent call last):
File "/home/admin/autobots/lib/python2.7/site-packages/autobase-0.0.1-py2.7.egg/autobase/service/server.py", line 90, in run
self.session.response(context)
File "/home/admin/autobots/lib/python2.7/site-packages/autobase-0.0.1-py2.7.egg/autobase/service/session.py", line 43, in response
self.request.finish()
File "/home/admin/autobots/lib/python2.7/site-packages/tornado-4.2-py2.7-linux-x86_64.egg/tornado/httputil.py", line 407, in finish
self.connection.finish()
File "/home/admin/autobots/lib/python2.7/site-packages/tornado-4.2-py2.7-linux-x86_64.egg/tornado/http1connection.py", line 459, in finish
self._pending_write.add_done_callback(self._finish_request)
File "/home/admin/autobots/lib/python2.7/site-packages/tornado-4.2-py2.7-linux-x86_64.egg/tornado/concurrent.py", line 243, in add_done_callback
fn(self)
File "/home/admin/autobots/lib/python2.7/site-packages/tornado-4.2-py2.7-linux-x86_64.egg/tornado/http1connection.py", line 491, in _finish_request
self.close()
File "/home/admin/autobots/lib/python2.7/site-packages/tornado-4.2-py2.7-linux-x86_64.egg/tornado/http1connection.py", line 297, in close
self.stream.close()
File "/home/admin/autobots/lib/python2.7/site-packages/tornado-4.2-py2.7-linux-x86_64.egg/tornado/iostream.py", line 427, in close
self.close_fd()
File "/home/admin/autobots/lib/python2.7/site-packages/tornado-4.2-py2.7-linux-x86_64.egg/tornado/iostream.py", line 995, in close_fd
self.socket.close()
AttributeError: 'NoneType' object has no attribute 'close'
]
Do I use Tornado in a wrong way? ask for help. thanks~

Tornado is not thread-safe by design. You can use a thread pool to do your own work, but you must call all Tornado methods (including in this case self.request.write and self.request.finish) from the IOLoop thread.

How to gracefully end SocketServer

I would like to stay in the std lib if possible. All of my threads run, and the sockets will listen appropriately. I need to be able to gracefully close the sockets, but I can't seem to get back to them to shutdown properly.
Problem is, when I get to server_close() it blows up on me (see exception trace output below).
Thanks for looking!
import sys
import threading
import SocketServer
class EchoRequestHandler(SocketServer.BaseRequestHandler):
def __init__(self, request, client_address, server):
SocketServer.BaseRequestHandler.__init__(self, request, client_address, server)
return
def setup(self):
return SocketServer.BaseRequestHandler.setup(self)
def handle(self):
# Echo the back to the client
data = self.request.recv(1024)
return
def finish(self):
return SocketServer.BaseRequestHandler.finish(self)
class EchoServer(SocketServer.TCPServer):
def __init__(self, server_address, handler_class=EchoRequestHandler):
SocketServer.TCPServer.__init__(self, server_address, handler_class)
return
def server_activate(self):
SocketServer.TCPServer.server_activate(self)
return
def serve_forever(self):
while True:
self.handle_request()
return
def server_close(self):
return SocketServer.TCPServer.server_close(self)
def bind_sockets(port_list):
server_list = []
for port in port_list:
server = EchoServer(("", int(port)), EchoRequestHandler)
server_thread = threading.Thread(target=server.serve_forever)
server_thread.setDaemon(True)
server_thread.start()
server_list.append(server)
print "listening on port", port
return server_list
def main():
port_list = [12345,54321]
active_servers = bind_sockets(port_list)
print "Hit Q to quit or R to reload."
while 1:
key_press = sys.stdin.read(1)
if key_press == "q":
sys.exit(2)
elif key_press == "r":
#We need to close gracefully so we can go on to do other stuff.
print active_servers.__len__()
for server in active_servers:
server.server_close()
print "closed"
if __name__ == "__main__":
main()
The exception stack-trace I get is:
Exception in thread Thread-1: Traceback (most recent call last):
File "~snip~/threading.py", line 810, in __bootstrap_inner self.run()
File "~snip~/threading.py", line 763, in run self.__target(*self.__args, **self.__kwargs)
File "kill_thread.py", line 36, in serve_forever self.handle_request()
File "~snip~/SocketServer.py", line 276, in handle_request fd_sets = _eintr_retry(select.select, [self], [], [], timeout)
File "~snip~/SocketServer.py", line 155, in _eintr_retry return func(*args) error: (9, 'Bad file descriptor')

You have two problems:
you need to call shutdown()
You shouldn't override server_forever()
like this:
# don't overrule server_forever, the default implementation
# handles the shutdown()
#def serve_forever(self):
# while True:
# self.handle_request()
# return
def server_close(self):
self.shutdown()
return SocketServer.TCPServer.server_close(self)