I've been messing around with PyQt and signals/slots across threads.
Here a situation where I can't find my mistake:
I have a class (MultipleProcessLauncher) that is able to launch multiple processes in separate threads.
I catch the stdout of each processes and send those messages to a single queue that is read by another thread (OutputWorker), this last thread should send a signal onNewMessage (I think it doesn't) catch on the main class but the callback function is never called.
The process threads populate the queue with messages
The reading thread catch all those messages (I can print them with print(item) in the while loop)
But:
- The signal of the reading thread doesn't seems to emit anything, so the callback function of the main thread is never called...
Your help would be much appreciated, I think I'm missing something with cross threads signals...
class OutputWorker(QObject):
onNewMessage = pyqtSignal(['QString'])
def __init__(self, queue, parent=None):
super(OutputWorker, self).__init__(parent)
self.queue = queue
def work(self):
while True:
item = self.queue.get()
self.onNewMessage.emit(item)
self.queue.task_done()
class MultipleProcessLauncher(QObject):
commandEvent = pyqtSignal(['QString'])
def __init__(self, parent=None):
super(MultipleProcessLauncher, self).__init__(parent)
self.messaging_queue = Queue()
# Start reading message
self.reading_thread = QThread()
self.worker = OutputWorker(self.messaging_queue)
self.worker.moveToThread(self.reading_thread)
self.worker.onNewMessage.connect(self.command_event)
self.reading_thread.started.connect(self.worker.work)
self.reading_thread.start()
def execute(self, command):
p = subprocess.Popen(command, stdout=subprocess.PIPE)
t = Thread(target=self.enqueue, args=(p.stdout, self.messaging_queue))
t.daemon = True
t.start()
def enqueue(self, stdout, queue):
for line in iter(stdout.readline, b''):
queue.put(line.decode())
stdout.close()
def command_event(self, event):
# This point is never reached
print('message received')
if __name__ == '__main__':
manager = MultipleProcessLauncher()
manager.execute('ipconfig')
time.sleep(100)
Qt's cross-thread signaling is based on event loop, so you need to exec a QApplication so that there is a main event loop to process signals from other threads. For example:
if __name__ == '__main__':
app = QApplication([])
manager = MultipleProcessLauncher()
manager.execute('ipconfig')
MAX_WAIT_MSEC = 100 * 1000 # 100 seconds
QTimer.singleShot(MAX_WAIT_MSEC, app.quit)
app.exec()
In your real application you will probably execute the manager based on user input so the execute would be in a slot, and there wouldn't be a need to quit, etc, but you get the idea.
Related
so I need to implement a following scenario:
- Several tasks are running simultaneously as processes.
- Each task should display a progress bar with a "Cancel" button, clicking on which should terminate it.
To achieve responsive GUI, I run the task for each process in a separate thread, and it seems that I need to create a separate wx.App for each process as well, otherwise the thread seems to be not running. This setup works fine, however:
a) I am not sure whether multiple wx.App's is a good idea or
b) If there is a better way of achieving my goal.
MWE below (note: in this sample code I could have used Update method of wx.ProgressDialog to identify whether the "Cancel" button has been pressed, but can't do so for my real application).
import wx, multiprocessing, time, psutil
from multiprocessing import Queue
from threading import Thread
from wx.lib.pubsub import pub as Publisher
#runs the task
def task_runner(q):
pid = multiprocessing.current_process().pid
q.put(pid)
while True:
print("Process Running")
time.sleep(1)
wx.CallAfter(Publisher.sendMessage, "update") #call to update the bar
class TestPanel():
def __init__(self,name):
self.q = Queue()
self.count=0
max = 80
# dialog to see progress and cancel the task
self.dlg = wx.GenericProgressDialog(name,
"An informative message",
maximum = max,
parent=None,
style = wx.PD_CAN_ABORT
| wx.PD_APP_MODAL
| wx.PD_ELAPSED_TIME
)
#set listener to dialog's "Cancel" button
for child in self.dlg.GetChildren():
if isinstance(child, wx.Button):
cancel_function = lambda evt, parent=self.dlg: self.onClose(evt, parent)
child.Bind(wx.EVT_BUTTON, cancel_function)
#subscribe to update the progress bar from the thread
Publisher.subscribe(self.updateProgress, "update")
# start thread which runs some task
p = Thread(target=task_runner, args=(self.q,))
p.start()
#updates the progress bar
def updateProgress(self):
print("updating progress")
self.count=self.count+10
self.dlg.Update(self.count)
#kills the process
def kill(self, proc_pid):
process = psutil.Process(proc_pid)
for proc in process.children(recursive=True):
proc.kill()
process.kill()
#closing the dialog event
def onClose(self, event, dialog):
""""""
print "Closing dialog!"
pid = self.q.get()
self.kill(pid)
dialog.Destroy()
# run process, each process creates its own wx.App
def runProcess(name):
app = wx.App(False)
TestPanel(name)
app.MainLoop()
# worker class to use for multiprocessing pool
class Worker():
def __call__(self, name):
return runProcess(name)
if __name__ == '__main__':
items=['Bar1', 'Bar2']
pool = multiprocessing.Pool(processes=2)
result = pool.map(Worker(), items) #create two processes
pool.close()
No, having more than one wx.App in a single process is not a good idea. Even creating a new one after the prior is finished can sometimes be problematic.
However since you are using multiprocess it is not quite the same. Unless I'm missing something, each OS process does have just one wx.App in your case, and since the parent process is not creating a wx.App then they are not trying to inherit that one (which would likely cause even more problems.)
I'm learning python multithreading and queues. The following creates a bunch of threads that pass data through a queue to another thread for printing:
import time
import threading
import Queue
queue = Queue.Queue()
def add(data):
return ["%sX" % x for x in data]
class PrintThread(threading.Thread):
def __init__(self, queue):
threading.Thread.__init__(self)
self.queue = queue
def run(self):
data = self.queue.get()
print data
self.queue.task_done()
class MyThread(threading.Thread):
def __init__(self, queue, data):
threading.Thread.__init__(self)
self.queue = queue
self.data = data
def run(self):
self.queue.put(add(self.data))
if __name__ == "__main__":
a = MyThread(queue, ["a","b","c"])
a.start()
b = MyThread(queue, ["d","e","f"])
b.start()
c = MyThread(queue, ["g","h","i"])
c.start()
printme = PrintThread(queue)
printme.start()
queue.join()
However, I see only the data from the first thread print out:
['aX', 'bX', 'cX']
Then nothing else, but the program doesn't exit. I have to kill the process to have it exit.
Ideally, after each MyThread does it data processing and puts the result to the queue, that thread should exit? Simultaneously the PrintThread should take whatever is on the queue and print it.
After all MyThread threads have finished and the PrintThread thread has finished processing everything on the queue, the program should exit cleanly.
What have I done wrong?
EDIT:
If each MyThread thread takes a while to process, is there a way to guarantee that the PrintThread thread will wait for all the MyThread threads to finish before it will exit itself?
That way the print thread will definitely have processed every possible data on the queue because all the other threads have already exited.
For example,
class MyThread(threading.Thread):
def __init__(self, queue, data):
threading.Thread.__init__(self)
self.queue = queue
self.data = data
def run(self):
time.sleep(10)
self.queue.put(add(self.data))
The above modification will wait for 10 seconds before putting anything on the queue. The print thread will run, but I think it's exiting too early since there is not data on the queue yet, so the program prints out nothing.
Your PrintThread does not loop but instead only prints out a single queue item and then stops running.
Therefore, the queue will never be empty and the queue.join() statement will prevent the main program from terminating
Change the run() method of your PrintThread into the following code in order to have all queue items processed:
try:
while True:
data = self.queue.get_nowait()
print data
self.queue.task_done()
except queue.Empty:
# All items have been taken off the queue
pass
I'm trying to figure out how to implement the concept of having my main thread spawn a new thread that processes data concurrently as messages are passed to it.
From what I figured so far the simplest way of doing this would be something like:
from PySide.QtCore import QCoreApplication, QObject, Signal, QThread, QTimer
class Foo(QObject):
do_foo = Signal(str)
def __init__(self, parent=None):
super(Foo, self).__init__(parent)
self.do_foo.connect(self._do_foo)
def foo(self, string):
self.do_foo.emit(string)
def _do_foo(self, string):
# Process string
print "Process thread:", self.thread().currentThreadId()
class App(QCoreApplication):
def run(self):
print "Main thread:", self.thread().currentThreadId()
thread = QThread()
foo = Foo()
foo.moveToThread(thread)
thread.start()
# Obviously the following should be done with the event-loop
# or at doing processing of events.
running = True
while running:
try:
string = raw_input()
foo.foo(string)
except EOFError:
running = False
thread.exit()
thread.wait()
self.exit()
if __name__ == '__main__':
import sys
app = App(sys.argv)
QTimer.singleShot(0, app.run)
sys.exit(app.exec_())
But if this would be the way of doing it I can not see what the use of Slots would be.
Or you can use the design patter "Provider-Consumer". How it works? Well you have to implement a queue. The spwaned thread will get the data from this queue while your main thread will feed the queue with new data.
Your spawned threads blocks while the queue is empty. This way you can even process data in more that one thread, and you don't have to worry about two threads trying to read the same data.
Here is some seudo-code for consumer threads.
class MyThread:
def __init__(self, queue):
self.queue = queue
self.event = Event() # I generally use threading.Event for stopping threads. You don't need it here.
def run():
while not self.event.isSet():
data = self.queue.get() # This stop the thread until new data be available.
do_something_with_data(data)
Then in your main thread:
import Queue
queue = Queue.Queue()
mthread = MyThread(queue)
mthread.start()
# And now you can send data to threads by:
queue.put(data)
What's the proper way to tell a looping thread to stop looping?
I have a fairly simple program that pings a specified host in a separate threading.Thread class. In this class it sleeps 60 seconds, the runs again until the application quits.
I'd like to implement a 'Stop' button in my wx.Frame to ask the looping thread to stop. It doesn't need to end the thread right away, it can just stop looping once it wakes up.
Here is my threading class (note: I haven't implemented looping yet, but it would likely fall under the run method in PingAssets)
class PingAssets(threading.Thread):
def __init__(self, threadNum, asset, window):
threading.Thread.__init__(self)
self.threadNum = threadNum
self.window = window
self.asset = asset
def run(self):
config = controller.getConfig()
fmt = config['timefmt']
start_time = datetime.now().strftime(fmt)
try:
if onlinecheck.check_status(self.asset):
status = "online"
else:
status = "offline"
except socket.gaierror:
status = "an invalid asset tag."
msg =("{}: {} is {}. \n".format(start_time, self.asset, status))
wx.CallAfter(self.window.Logger, msg)
And in my wxPyhton Frame I have this function called from a Start button:
def CheckAsset(self, asset):
self.count += 1
thread = PingAssets(self.count, asset, self)
self.threads.append(thread)
thread.start()
Threaded stoppable function
Instead of subclassing threading.Thread, one can modify the function to allow
stopping by a flag.
We need an object, accessible to running function, to which we set the flag to stop running.
We can use threading.currentThread() object.
import threading
import time
def doit(arg):
t = threading.currentThread()
while getattr(t, "do_run", True):
print ("working on %s" % arg)
time.sleep(1)
print("Stopping as you wish.")
def main():
t = threading.Thread(target=doit, args=("task",))
t.start()
time.sleep(5)
t.do_run = False
if __name__ == "__main__":
main()
The trick is, that the running thread can have attached additional properties. The solution builds
on assumptions:
the thread has a property "do_run" with default value True
driving parent process can assign to started thread the property "do_run" to False.
Running the code, we get following output:
$ python stopthread.py
working on task
working on task
working on task
working on task
working on task
Stopping as you wish.
Pill to kill - using Event
Other alternative is to use threading.Event as function argument. It is by
default False, but external process can "set it" (to True) and function can
learn about it using wait(timeout) function.
We can wait with zero timeout, but we can also use it as the sleeping timer (used below).
def doit(stop_event, arg):
while not stop_event.wait(1):
print ("working on %s" % arg)
print("Stopping as you wish.")
def main():
pill2kill = threading.Event()
t = threading.Thread(target=doit, args=(pill2kill, "task"))
t.start()
time.sleep(5)
pill2kill.set()
t.join()
Edit: I tried this in Python 3.6. stop_event.wait() blocks the event (and so the while loop) until release. It does not return a boolean value. Using stop_event.is_set() works instead.
Stopping multiple threads with one pill
Advantage of pill to kill is better seen, if we have to stop multiple threads
at once, as one pill will work for all.
The doit will not change at all, only the main handles the threads a bit differently.
def main():
pill2kill = threading.Event()
tasks = ["task ONE", "task TWO", "task THREE"]
def thread_gen(pill2kill, tasks):
for task in tasks:
t = threading.Thread(target=doit, args=(pill2kill, task))
yield t
threads = list(thread_gen(pill2kill, tasks))
for thread in threads:
thread.start()
time.sleep(5)
pill2kill.set()
for thread in threads:
thread.join()
This has been asked before on Stack. See the following links:
Is there any way to kill a Thread in Python?
Stopping a thread after a certain amount of time
Basically you just need to set up the thread with a stop function that sets a sentinel value that the thread will check. In your case, you'll have the something in your loop check the sentinel value to see if it's changed and if it has, the loop can break and the thread can die.
I read the other questions on Stack but I was still a little confused on communicating across classes. Here is how I approached it:
I use a list to hold all my threads in the __init__ method of my wxFrame class: self.threads = []
As recommended in How to stop a looping thread in Python? I use a signal in my thread class which is set to True when initializing the threading class.
class PingAssets(threading.Thread):
def __init__(self, threadNum, asset, window):
threading.Thread.__init__(self)
self.threadNum = threadNum
self.window = window
self.asset = asset
self.signal = True
def run(self):
while self.signal:
do_stuff()
sleep()
and I can stop these threads by iterating over my threads:
def OnStop(self, e):
for t in self.threads:
t.signal = False
I had a different approach. I've sub-classed a Thread class and in the constructor I've created an Event object. Then I've written custom join() method, which first sets this event and then calls a parent's version of itself.
Here is my class, I'm using for serial port communication in wxPython app:
import wx, threading, serial, Events, Queue
class PumpThread(threading.Thread):
def __init__ (self, port, queue, parent):
super(PumpThread, self).__init__()
self.port = port
self.queue = queue
self.parent = parent
self.serial = serial.Serial()
self.serial.port = self.port
self.serial.timeout = 0.5
self.serial.baudrate = 9600
self.serial.parity = 'N'
self.stopRequest = threading.Event()
def run (self):
try:
self.serial.open()
except Exception, ex:
print ("[ERROR]\tUnable to open port {}".format(self.port))
print ("[ERROR]\t{}\n\n{}".format(ex.message, ex.traceback))
self.stopRequest.set()
else:
print ("[INFO]\tListening port {}".format(self.port))
self.serial.write("FLOW?\r")
while not self.stopRequest.isSet():
msg = ''
if not self.queue.empty():
try:
command = self.queue.get()
self.serial.write(command)
except Queue.Empty:
continue
while self.serial.inWaiting():
char = self.serial.read(1)
if '\r' in char and len(msg) > 1:
char = ''
#~ print('[DATA]\t{}'.format(msg))
event = Events.PumpDataEvent(Events.SERIALRX, wx.ID_ANY, msg)
wx.PostEvent(self.parent, event)
msg = ''
break
msg += char
self.serial.close()
def join (self, timeout=None):
self.stopRequest.set()
super(PumpThread, self).join(timeout)
def SetPort (self, serial):
self.serial = serial
def Write (self, msg):
if self.serial.is_open:
self.queue.put(msg)
else:
print("[ERROR]\tPort {} is not open!".format(self.port))
def Stop(self):
if self.isAlive():
self.join()
The Queue is used for sending messages to the port and main loop takes responses back. I've used no serial.readline() method, because of different end-line char, and I have found the usage of io classes to be too much fuss.
Depends on what you run in that thread.
If that's your code, then you can implement a stop condition (see other answers).
However, if what you want is to run someone else's code, then you should fork and start a process. Like this:
import multiprocessing
proc = multiprocessing.Process(target=your_proc_function, args=())
proc.start()
now, whenever you want to stop that process, send it a SIGTERM like this:
proc.terminate()
proc.join()
And it's not slow: fractions of a second.
Enjoy :)
My solution is:
import threading, time
def a():
t = threading.currentThread()
while getattr(t, "do_run", True):
print('Do something')
time.sleep(1)
def getThreadByName(name):
threads = threading.enumerate() #Threads list
for thread in threads:
if thread.name == name:
return thread
threading.Thread(target=a, name='228').start() #Init thread
t = getThreadByName('228') #Get thread by name
time.sleep(5)
t.do_run = False #Signal to stop thread
t.join()
I find it useful to have a class, derived from threading.Thread, to encapsulate my thread functionality. You simply provide your own main loop in an overridden version of run() in this class. Calling start() arranges for the object’s run() method to be invoked in a separate thread.
Inside the main loop, periodically check whether a threading.Event has been set. Such an event is thread-safe.
Inside this class, you have your own join() method that sets the stop event object before calling the join() method of the base class. It can optionally take a time value to pass to the base class's join() method to ensure your thread is terminated in a short amount of time.
import threading
import time
class MyThread(threading.Thread):
def __init__(self, sleep_time=0.1):
self._stop_event = threading.Event()
self._sleep_time = sleep_time
"""call base class constructor"""
super().__init__()
def run(self):
"""main control loop"""
while not self._stop_event.isSet():
#do work
print("hi")
self._stop_event.wait(self._sleep_time)
def join(self, timeout=None):
"""set stop event and join within a given time period"""
self._stop_event.set()
super().join(timeout)
if __name__ == "__main__":
t = MyThread()
t.start()
time.sleep(5)
t.join(1) #wait 1s max
Having a small sleep inside the main loop before checking the threading.Event is less CPU intensive than looping continuously. You can have a default sleep time (e.g. 0.1s), but you can also pass the value in the constructor.
Sometimes you don't have control over the running target. In those cases you can use signal.pthread_kill to send a stop signal.
from signal import pthread_kill, SIGTSTP
from threading import Thread
from itertools import count
from time import sleep
def target():
for num in count():
print(num)
sleep(1)
thread = Thread(target=target)
thread.start()
sleep(5)
pthread_kill(thread.ident, SIGTSTP)
result
0
1
2
3
4
[14]+ Stopped
I am developing an application that uses multiple threads to gather data from a variety of network devices. I'm using PyQT to display the collected data on a GUI. I am using regular python threads (from thread, threading) in my app (instead of QThread). In order to update the GUI on the different threads, I use a lock (thread.allocate_lock()). So, anytime a GUI update will happen, I call with lock, update GUI. Any concerns about this?
I'm pretty sure that updating the GUI from different threads is dangerous in Qt, even if you try to lock things in your own code. For one thing, Qt might be doing its own event processing on the main thread, and it will not acquire your lock to protect objects that it might modify. On this page in the Qt docs, the fact that QWidget is not reentrant or thread-safe is explicitly mentioned.
I recommend that you post the collected data, or a processed version of it, back to the main thread. Use a queued signal/slot connection, or a custom QEvent and QApplication::postEvent to do this. In the previous question that jkerian mentions, it says that you'll have to use QThread instead of python's threads if you want event posting to work correctly.
This is a late reply but I wanted to share what I found. This is code from WickedDevice Blog that I found useful to understand threads and PyQt:
#authors: Dirk Swart, Doudewijn Rempt, Jacob Hallen
import sys, time, threading, random, Queue
from PyQt4 import QtGui, QtCore as qt
import serial
SERIALPORT = 'COM6'
class GuiPart(QtGui.QMainWindow):
def __init__(self, queue, endcommand, *args):
QtGui.QMainWindow.__init__(self, *args)
self.setWindowTitle('Arduino Serial Demo')
self.queue = queue
# We show the result of the thread in the gui, instead of the console
self.editor = QtGui.QTextEdit(self)
self.setCentralWidget(self.editor)
self.endcommand = endcommand
def closeEvent(self, ev):
self.endcommand()
def processIncoming(self):
"""
Handle all the messages currently in the queue (if any).
"""
while self.queue.qsize():
try:
msg = self.queue.get(0)
# Check contents of message and do what it says
# As a test, we simply print it
self.editor.insertPlainText(str(msg))
except Queue.Empty:
pass
class ThreadedClient:
"""
Launch the main part of the GUI and the worker thread. periodicCall and
endApplication could reside in the GUI part, but putting them here
means that you have all the thread controls in a single place.
"""
def __init__(self):
# Create the queue
self.queue = Queue.Queue()
# Set up the GUI part
self.gui=GuiPart(self.queue, self.endApplication)
self.gui.show()
# A timer to periodically call periodicCall :-)
self.timer = qt.QTimer()
qt.QObject.connect(self.timer,
qt.SIGNAL("timeout()"),
self.periodicCall)
# Start the timer -- this replaces the initial call to periodicCall
self.timer.start(100)
# Set up the thread to do asynchronous I/O
# More can be made if necessary
self.running = 1
self.thread1 = threading.Thread(target=self.workerThread1)
self.thread1.start()
def periodicCall(self):
"""
Check every 100 ms if there is something new in the queue.
"""
self.gui.processIncoming()
if not self.running:
root.quit()
def endApplication(self):
self.running = 0
def workerThread1(self):
"""
This is where we handle the asynchronous I/O.
Put your stuff here.
"""
while self.running:
#This is where we poll the Serial port.
#time.sleep(rand.random() * 0.3)
#msg = rand.random()
#self.queue.put(msg)
ser = serial.Serial(SERIALPORT, 115200)
msg = ser.readline();
if (msg):
self.queue.put(msg)
else: pass
ser.close()
if __name__ == "__main__":
#rand = random.Random()
root = QtGui.QApplication(sys.argv)
client = ThreadedClient()
sys.exit(app.exec_())
I use pyqtSignal and Python's threading. You can create threads and when the thread is completed have it send a signal to update your GUI.