Develop Reference

Python multithreading- memory leak when use an shared object(so)

I have a python programs that gets memory leaks when use an third-party SO.
I simplify my code like this:
import time
import sys
import threading
import codecs
import ctypes
sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach())
class TestThirdPartySo(object):
def __init__(self):
# this so uses thread-specific data
self.call_stat_so = ctypes.CDLL("./third_party_fun.so")
self.handle = self.call_stat_so._handle
def test_fun(self):
self.call_stat_so.fun_xxx()
def thread_fun():
TestThirdPartySo().test_fun()
def test_main(num):
count = 0
while True:
# create 3 * num threads
thread_num = 3
thread_list = []
for _ in range(thread_num):
thread_list.append(threading.Thread(target=thread_fun))
for thread in thread_list:
thread.start()
for thread in thread_list:
thread.join()
count += thread_num
time.sleep(0.01)
if count % 100 == 0:
print("finied %s" % count)
if count > num:
break
print("end !!!!")
if __name__ == '__main__':
num = sys.argv[1]
test_main(int(num))
Now, I know this shared object uses thread-specific data.And I have tried to close the SO after called it like this:
class TestThirdPartySo(object):
def __init__(self):
# this so uses thread-specific data
self.call_stat_so = ctypes.CDLL("./third_party_fun.so")
self.handle = self.call_stat_so._handle
def test_fun(self):
self.call_stat_so.fun_xxx()
def __del__(self):
dlclose_func(self.handle)
def dlclose_func(_handle):
dlclose_func_tmp = ctypes.cdll.LoadLibrary('libdl.so').dlclose
dlclose_func_tmp.argtypes = [ctypes.c_void_p]
dlclose_func_tmp(_handle)
But I failed to close the so. And I'm also not sure if the leaked memory will be freed after closing the so.
If the program not uses multi-threads or creates a fixed number of threads(threadpool), it works ok.
For some reason,I need create threads constantly in my program. What can I do to prevent this memory leaks?

How can I properly call a method from its callback method within a class?

I Have a code with two functions. Function 'send_thread' and Function 'receive_thread' which is the callback of 'send_thread'. What I want to do is to run 'send_thread', this activates 'receive_thread' and once it's over repeat it all again. To do so, I have come up with the code below. This is not giving the desired results, since the 'send_thread' gets called again but doesn't activate the callback anymore. Thank you in advance for your help.
I have noticed, that the function gets called at the end of the receive_thread and runs for the amount of time that I wait in the send_thread (rospy.sleep()). I does never activate the callback again after the first try though.
import rospy
import pepper_2d_simulator
import threading
class TROS(object):
def __init__(self):
self.cmd_vel_pub = rospy.Publisher('cmd_vel',Twist)
self.event = threading.Event()
def send_thread(self):
#send commmand
self.event.set()
sequence = [[1,0,0.05],[0,0,0],[0,0,0.1292]]
for cmd in sequence:
rospy.Rate(0.5).sleep()
msg = Twist()
msg.linear.x = cmd[0]
msg.linear.y = cmd[1]
msg.angular.z = cmd[2]
t = rospy.get_rostime()
self.cmd_vel_pub.publish(msg)
self.event.clear()
rospy.sleep(1)
def receive_thread(self,msg):
#if something is being send, listen to this
if self.event.isSet():
frame_id = msg.header.frame_id
self.x_odom = msg.pose.pose.position.x
self.y_odom = msg.pose.pose.position.y
self.z_odom = msg.pose.pose.position.z
self.pos_odom = [self.x_odom,self.y_odom,self.z_odom,1]
self.ang_odom = msg.pose.pose.orientation.z
self.time = msg.header.stamp.secs + msg.header.stamp.nsecs
#some transformations here to get self.trans...
else:
#after self.event() is cleared, rename and run again
self.x_odom = self.trans_br_x
self.y_odom = self.trans_br_y
self.ang_odom = self.rot_br_ang
self.send_thread()
def init_node(self):
rospy.init_node('pepper_cmd_evaluator',anonymous = True)
rospy.Subscriber('odom',Odometry,self.receive_thread)
if __name__ == '__main__':
thinking = Thinking()
thinking.init_node()
thinking.send_thread()
The expected result is that I am able to loop this two function so that I call send_thread, this activates receive thread. Then send_thread stops, receive_thread stops and activates the send_thread again. I want to do this 10 times.

I have now figured out how to this. I will post my solution in case anyone else runs into a similar problem. The working solution I came up with is pretty simple. I created a self.flag variable and alternatively set it to True and False in the send_thread and callback respectively. The code:
import rospy
import pepper_2d_simulator
import threading
class TROS(object):
def __init__(self):
self.cmd_vel_pub = rospy.Publisher('cmd_vel',Twist)
self.event = threading.Event()
self.count = 0
self.flag = True
def send_thread(self):
while self.count < 10:
if self.flag:
self.count = self.count + 1
#send commmand
self.event.set()
sequence = [[1,0,0.05],[0,0,0],[0,0,0.1292]]
for cmd in sequence:
rospy.Rate(0.5).sleep()
msg = Twist()
msg.linear.x = cmd[0]
msg.linear.y = cmd[1]
msg.angular.z = cmd[2]
t = rospy.get_rostime()
self.cmd_vel_pub.publish(msg)
self.event.clear()
rospy.sleep(0.3)
self.flag = False
rospy.signal_shutdown('Command finished')
def receive_thread(self,msg):
#if something is being send, listen to this
if self.event.isSet():
frame_id = msg.header.frame_id
self.x_odom = msg.pose.pose.position.x
self.y_odom = msg.pose.pose.position.y
self.z_odom = msg.pose.pose.position.z
self.pos_odom = [self.x_odom,self.y_odom,self.z_odom,1]
self.ang_odom = msg.pose.pose.orientation.z
self.time = msg.header.stamp.secs + msg.header.stamp.nsecs
#some transformations here to get self.trans...
else:
#after self.event() is cleared, rename and run again
self.x_odom = self.trans_br_x
self.y_odom = self.trans_br_y
self.ang_odom = self.rot_br_ang
self.flag = True
def init_node(self):
rospy.init_node('pepper_cmd_evaluator',anonymous = True)
rospy.Subscriber('odom',Odometry,self.receive_thread)
if __name__ == '__main__':
thinking = Thinking()
thinking.init_node()
thinking.send_thread()

Worker process is not being called in simple example?

Here's my simple example code:
# -*- coding: utf-8 -*-
import multiprocessing
import time
def some_function():
return "hello!"
def get_address_type(address):
return 'tcp'
class PseudoThing():
def __init__(self, address):
self.address = address
self.server = None
def attach(self):
if get_address_type(self.address) == 'tcp':
ip, port = self.address.split(':', 1)
self.server = some_function()
else:
self.server = some_function()
def worker(self):
print("in worker!")
for i in range(10):
print("I'm a worker doing worker things...")
self.attach()
if self.server:
return True
time.sleep(0.2)
def parent(self):
print("I'm a parent doing parent things!")
def start_process(self):
p = multiprocessing.Process(target=self.worker)
print("starting process")
p.start()
return p
def main():
nvr = PseudoThing("kitty")
p = nvr.start_process()
p.join()
print("__name__ = {}".format(__name__))
if __name__ == "__main__":
print("doing main!")
main()
However, nothing is getting printed out from worker, as I would expect. Instead, this is the output of the program:
__name__ = __main__
doing main!
starting process
I am suspecting this has something to do with the class structure (I recall having issues with this on Python 2), but I am not entirely sure. Where can I add more print statements to figure what's going on? Where are things going wrong?

How to pause a thread (python)

The context:
I'm building a Graphical Interface with Qt creator and the "behaviour" file in python. A test version of my GUI is:
The expected behaviour:
I am running 2 different threads which are referred to the same function with different input arguments. With the SELECTOR button I can assign the value of 1 or 2 to a variable (and display it)
The button Start thread enables the correct thread to start (the first time).
The loop should be turned off by the stop button by modifying the global running variable.
This is my code
# -*- coding: utf-8 -*-
from PyQt4 import QtCore, QtGui, uic
import sys
import threading
import time
import Queue
running = False
first_thread = None
second_thread = None
form_class = uic.loadUiType("simple2.ui")[0]
q = Queue.Queue()
select = 0
def action(string, queue): #function called by threads
global running
while(running):
phrase = string
if queue.qsize() < 10:
queue.put(phrase)
#else:
# print queue.qsize()
class MyWindowClass(QtGui.QMainWindow, form_class):
def __init__(self, parent=None):
QtGui.QMainWindow.__init__(self, parent)
self.setupUi(self)
#buttons
self.startButton.clicked.connect(self.start_clicked)
self.stopButton.clicked.connect(self.stop_clicked)
self.selector.clicked.connect(self.sel_click)
#variables
self.first = False
self.second = False
#queue
self.timer = QtCore.QTimer(self)
self.timer.timeout.connect(self.update_phrase)
self.timer.start(1)
def start_clicked(self): #start button callback
global select
if select > 0:
global running
running = True
print "started"
if (not self.first) & (select == 1):
first_thread.start()
self.first = True
if (not self.second) & (select == 2):
second_thread.start()
self.second = True
self.startButton.setEnabled(False)
self.startButton.setText('Starting...')
def stop_clicked(self): #stop button callback
global running
running = False
print "stopped"
self.startButton.setEnabled(True)
self.startButton.setText('Start Thread')
def sel_click(self): #selector button callback
global select
if select < 2:
select = select + 1
else:
select = 1
self.thread_counter.setText(str(select))
def update_phrase(self): #looping function
global running
if (not q.empty()) & running:
self.startButton.setText('Thread on')
abc = q.get()
print abc
def closeEvent(self, event):
global running
running = False
if __name__ == "__main__":
first_thread = threading.Thread(target=action, args = ("first", q))
second_thread = threading.Thread(target=action, args = ("second", q))
app = QtGui.QApplication(sys.argv)
w = MyWindowClass(None)
w.setWindowTitle('Multiple threads test in python')
w.show()
app.exec_()
For now, each thread should simple print on terminal their arguments ("First" or "Second").
If threads are started for the first time, my code works. But I would like to switch between threads infinite times.
Since threads cannot be stopped, is there a way to "pause" them?
I cannot find a solution, I hope someone will help me also with a piece of code. Thank you in advance

You can use Lock class to do that, a simple example would be:
import threading
lock = threading.Lock()
//here it will be lock
lock.acquire() # will block if lock is already held
...
then in other side do
//this will wake up
lock.release()
you can read more here http://effbot.org/zone/thread-synchronization.htm

Multithreading (?): Manual interference in a loop

I've been looking into a way to directly change variables in a running module.
What I want to achieve is that a load test is being run and that I can manually adjust the call pace or whatsoever.
Below some code that I just created (not-tested e.d.), just to give you an idea.
class A():
def __init__(self):
self.value = 1
def runForever(self):
while(1):
print self.value
def setValue(self, value):
self.value = value
if __name__ == '__main__':
#Some code to create the A object and directly apply the value from an human's input
a = A()
#Some parallelism or something has to be applied.
a.runForever()
a.setValue(raw_input("New value: "))
Edit #1: Yes, I know that now I will never hit the a.setValue() :-)

Here is a multi-threaded example. This code will work with the python interpreter but not with the Python Shell of IDLE, because the raw_input function is not handled the same way.
from threading import Thread
from time import sleep
class A(Thread):
def __init__(self):
Thread.__init__(self)
self.value = 1
self.stop_flag = False
def run(self):
while not self.stop_flag:
sleep(1)
print(self.value)
def set_value(self, value):
self.value = value
def stop(self):
self.stop_flag = True
if __name__ == '__main__':
a = A()
a.start()
try:
while 1:
r = raw_input()
a.set_value(int(r))
except:
a.stop()

The pseudo code you wrote is quite similar to the way Threading / Multiprocessing works in python. You will want to start a (for example) thread that "runs forever" and then instead of modifying the internal rate value directly, you will probably just send a message through a Queue that gives the new value.
Check out this question.
Here is a demonstration of doing what you asked about. I prefer to use Queues to directly making calls on threads / processes.
import Queue # !!warning. if you use multiprocessing, use multiprocessing.Queue
import threading
import time
def main():
q = Queue.Queue()
tester = Tester(q)
tester.start()
while True:
user_input = raw_input("New period in seconds or (q)uit: ")
if user_input.lower() == 'q':
break
try:
new_speed = float(user_input)
except ValueError:
new_speed = None # ignore junk
if new_speed is not None:
q.put(new_speed)
q.put(Tester.STOP_TOKEN)
class Tester(threading.Thread):
STOP_TOKEN = '<<stop>>'
def __init__(self, q):
threading.Thread.__init__(self)
self.q = q
self.speed = 1
def run(self):
while True:
# get from the queue
try:
item = self.q.get(block=False) # don't hang
except Queue.Empty:
item = None # do nothing
if item:
# stop when requested
if item == self.STOP_TOKEN:
break # stop this thread loop
# otherwise check for a new speed
try:
self.speed = float(item)
except ValueError:
pass # whatever you like with unknown input
# do your thing
self.main_code()
def main_code(self):
time.sleep(self.speed) # or whatever you want to do
if __name__ == '__main__':
main()