Develop Reference

How to check periodically if a thread is finished

I am trying to run a script which runs Asynchronously using threadings. I have run into an issue on how to check periodically if a thread is still alive (under start_thread1). I don't want to use join() as that will freeze the GUI until the threads is finished.
If that is not possible, I am open-minded with any other ways of doing it.
Here is the code I am using - this is a part of part of the code just to outline the "issue" that I have:
from tkinter.constants import LEFT, RIGHT, S
import tkinter.messagebox
from matplotlib import pyplot as plt
import tkinter as tk, time, threading, random, queue
class GuiPart(object):
def __init__(self, master, queue, queue2, client_instance):
self.queue = queue
self.queue2 = queue2
self.x = []
self.y= []
# Set up the GUI
self.Button2 = tk.Button(master, text="Button2", padx=10,
pady=5, fg="white", bg="#263D42", command=client_instance.start_thread1)
self.Button2.pack(side = RIGHT)
def processIncoming(self):
""" Handle all messages currently in the queue, if any. """
while not self.queue.empty():
msg = self.queue.get_nowait()
self.x.append(msg)
print(msg)
while not self.queue2.empty():
msg2 = self.queue2.get_nowait()
self.y.append(msg2)
fig, ax = plt.subplots()
ax.plot(self.x, self.y)
plt.show()
class ThreadedClient(object):
"""
Launch the main part of the GUI and the worker thread. periodic_call()
and end_application() could reside in the GUI part, but putting them
here means that you have all the thread controls in a single place.
"""
def __init__(self, master):
"""
Start the GUI and the asynchronous threads. We are in the main
(original) thread of the application, which will later be used by
the GUI as well. We spawn a new thread for the worker (I/O).
"""
self.master = master
# Create the queue
self.queue = queue.Queue()
self.queue2 = queue.Queue()
self.running=True
# Set up the GUI part
self.gui = GuiPart(master, self.queue, self.queue2, self)
# Set up the thread to do asynchronous I/O
# More threads can also be created and used, if necessary
def start_thread1(self):
thread1=threading.Thread(target=self.worker_thread1)
thread1.start()
# how to check periodically if the thread is finished and when is finished run self.gui.processIncoming()
# if I run it straight away like this the self.gui.processIncoming() will run before the thread will finish and nothing will be plotted
if thread1.is_alive() == False:
self.gui.processIncoming()
def worker_thread1(self):
"""
This is where we handle the asynchronous I/O. For example, it may be
a 'select()'. One important thing to remember is that the thread has
to yield control pretty regularly, be it by select or otherwise.
"""
if self.running:
time.sleep(5) # I am using time.sleep(5) just to simulate a long-running process
for i in range(20):
msg = i
self.queue.put(msg)
for j in range(20):
msg2 = j
self.queue2.put(msg2)
root = tk.Tk()
root.title('Matplotlib threading')
client = ThreadedClient(root)
root.mainloop()

If I understand correctly, you have a tkinter program with a second thread that does some data collection, and you need to get data from that second thread back into the gui. You can't simply wait for the second thread to finish because that would block the tkinter main loop.
One solution is to create a callback function, pass it to the second thread, and call it as the very last step in the second thread. Most tkinter objects aren't threadsafe, so if you're going to update the GUI in the callback function, you have to run the callback in the main thread. To do this, base the callback on tkinter's after_idle function. This causes the callback to occur in tk's event loop, in the main thread, much like a tkinter event handler.
This program does that, and is similar to your program. I changed a few minor things to make my static type checker (pylint) happy. I don't use matplotlib so I took that code out.
The important stuff is in start_thread1. The function f is declared and passed as an argument to the thread. Note that f doesn't call processIncoming, but passes it to after_idle; that instructs the tk main loop to perform the actual call. The function that got passed to worker_thread1 is called as the last step in the thread.
The end result is that processIncoming() is fired into the main thread when the worker thread finishes.
from tkinter.constants import RIGHT
import tkinter as tk
import time
import threading
import queue
class GuiPart:
def __init__(self, master, queue1, queue2, client_instance):
self.queue1 = queue1
self.queue2 = queue2
self.x = []
self.y= []
# Set up the GUI
self.Button2 = tk.Button(master, text="Button2", padx=10,
pady=5, fg="white", bg="#263D42",
command=client_instance.start_thread1)
self.Button2.pack(side = RIGHT)
def processIncoming(self):
""" Handle all messages currently in the queue, if any. """
print(threading.current_thread())
while not self.queue1.empty():
msg = self.queue1.get_nowait()
self.x.append(msg)
print("X", msg)
while not self.queue2.empty():
msg2 = self.queue2.get_nowait()
self.y.append(msg2)
print("Y", msg2)
print("Make a plot now")
class ThreadedClient:
"""
Launch the main part of the GUI and the worker thread. periodic_call()
and end_application() could reside in the GUI part, but putting them
here means that you have all the thread controls in a single place.
"""
def __init__(self, master):
"""
Start the GUI and the asynchronous threads. We are in the main
(original) thread of the application, which will later be used by
the GUI as well. We spawn a new thread for the worker (I/O).
"""
self.master = master
# Create the queue
self.queue1 = queue.Queue()
self.queue2 = queue.Queue()
self.running=True
# Set up the GUI part
self.gui = GuiPart(master, self.queue1, self.queue2, self)
# Set up the thread to do asynchronous I/O
# More threads can also be created and used, if necessary
def start_thread1(self):
def f():
self.master.after_idle(self.gui.processIncoming)
thread1=threading.Thread(target=self.worker_thread1, args=(f, ))
thread1.start()
def worker_thread1(self, callback):
"""
This is where we handle the asynchronous I/O. For example, it may be
a 'select()'. One important thing to remember is that the thread has
to yield control pretty regularly, be it by select or otherwise.
"""
print(threading.current_thread())
if self.running:
time.sleep(1) # simulate a long-running process
for i in range(20):
msg = i
self.queue1.put(msg)
for j in range(20):
msg2 = j
self.queue2.put(msg2)
callback()
root = tk.Tk()
root.title('Matplotlib threading')
client = ThreadedClient(root)
root.mainloop()

Freezing/Hanging tkinter GUI in waiting for the thread to complete

My interface is freezing on pressing the button. I am using threading but I am not sure why is still hanging. Any help will be appreciated. Thanks in advance
class magic:
def __init__(self):
self.mainQueue=queue.Queue()
def addItem(self,q):
self.mainQueue.put(q)
def startConverting(self,funcName):
if(funcName=="test"):
while not self.mainQueue.empty():
t = Thread(target = self.threaded_function)
t.start()
t.join()
def threaded_function(self):
time.sleep(5)
print(self.mainQueue.get())
m=magic()
def helloCallBack():
m.addItem("asd")
m.startConverting("test") //this line of code is freezing
B = tkinter.Button(top, text ="Hello", command = helloCallBack)
B.pack()
top.mainloop()

Here's a recipe for doing an asynchronous task with a tkinter-based GUI. I adapted it from a recipe in the cited book. You should be able to modify it to do what you need.
To keep the GUI responsive requires not interfering with its mainloop() by doing something like join()ing a background thread—which makes the GUI "hang" until the thread is finished. This is accomplished by using the universal after() widget method to poll a Queue at regular intervals.
# from "Python Coobook 2nd Edition", section 11.9, page 439.
# Modified to work in Python 2 & 3.
from __future__ import print_function
try:
import Tkinter as tk, time, threading, random, Queue as queue
except ModuleNotFoundError: # Python 3
import tkinter as tk, time, threading, random, queue
class GuiPart(object):
def __init__(self, master, queue, end_command):
self.queue = queue
# Set up the GUI
tk.Button(master, text='Done', command=end_command).pack()
# Add more GUI stuff here depending on your specific needs
def processIncoming(self):
""" Handle all messages currently in the queue, if any. """
while self.queue.qsize():
try:
msg = self.queue.get_nowait()
# Check contents of message and do whatever is needed. As a
# simple example, let's print it (in real life, you would
# suitably update the GUI's display in a richer fashion).
print(msg)
except queue.Empty:
# just on general principles, although we don't expect this
# branch to be taken in this case, ignore this exception!
pass
class ThreadedClient(object):
"""
Launch the main part of the GUI and the worker thread. periodic_call()
and end_application() could reside in the GUI part, but putting them
here means that you have all the thread controls in a single place.
"""
def __init__(self, master):
"""
Start the GUI and the asynchronous threads. We are in the main
(original) thread of the application, which will later be used by
the GUI as well. We spawn a new thread for the worker (I/O).
"""
self.master = master
# Create the queue
self.queue = queue.Queue()
# Set up the GUI part
self.gui = GuiPart(master, self.queue, self.end_application)
# Set up the thread to do asynchronous I/O
# More threads can also be created and used, if necessary
self.running = True
self.thread1 = threading.Thread(target=self.worker_thread1)
self.thread1.start()
# Start the periodic call in the GUI to check the queue
self.periodic_call()
def periodic_call(self):
""" Check every 200 ms if there is something new in the queue. """
self.master.after(200, self.periodic_call)
self.gui.processIncoming()
if not self.running:
# This is the brutal stop of the system. You may want to do
# some cleanup before actually shutting it down.
import sys
sys.exit(1)
def worker_thread1(self):
"""
This is where we handle the asynchronous I/O. For example, it may be
a 'select()'. One important thing to remember is that the thread has
to yield control pretty regularly, be it by select or otherwise.
"""
while self.running:
# To simulate asynchronous I/O, create a random number at random
# intervals. Replace the following two lines with the real thing.
time.sleep(rand.random() * 1.5)
msg = rand.random()
self.queue.put(msg)
def end_application(self):
self.running = False # Stops worker_thread1 (invoked by "Done" button).
rand = random.Random()
root = tk.Tk()
client = ThreadedClient(root)
root.mainloop()

For anyone having a problem with sys.exit(1) in #martineau's code - if you replace sys.exit(1) with self.master.destroy() the program ends gracefully. I lack the reputation to add a comment, hence the seperate answer.

Python multiprocessing asynchronous callback

I'm writing a program with a GUI using TKinter, in which the user can click a button and a new process is started to perform work using multiprocess.Process. This is necessary so the GUI can still be used while the work is being done, which can take several seconds.
The GUI also has a text box where the status of the program is displayed when things happen. This is often straight forward, with each function calling an add_text() function which just prints text in the text box. However, when add_text() is called in the separate process, the text does not end up in the text box.
I've thought about using a Pipe or Queue, but that would require using some sort of loop to check if anything has been returned from the process and that would also cause the main (GUI) process to be unusable. Is there some way to call a function in one process that will do work in another?
Here's an simple example of what I'm trying to do
import time
import multiprocessing as mp
import tkinter as tk
textbox = tk.Text()
def add_text(text):
# Insert text into textbox
textbox.insert(tk.END, text)
def worker():
x = 0
while x < 10:
add_text('Sleeping for {0} seconds'.format(x)
x += 1
time.sleep(1)
proc = mp.Process(target=worker)
# Usually happens on a button click
proc.start()
# GUI should still be usable here

The asyncronous things actually require loop.
You could attach function to the TkInter's loop by using Tk.after() method.
import Tkinter as tk
class App():
def __init__(self):
self.root = tk.Tk()
self.check_processes()
self.root.mainloop()
def check_processes(self):
if process_finished:
do_something()
else:
do_something_else()
self.after(1000, check_processes)
app=App()

I ended up using a multiprocessing.Pipe by using TKinter's after() method to perform the looping. It loops on an interval and checks the pipe to see if there's any messages from the thread, and if so it inserts them into the text box.
import tkinter
import multiprocessing
def do_something(child_conn):
while True:
child_conn.send('Status text\n')
class Window:
def __init__(self):
self.root = tkinter.Tk()
self.textbox = tkinter.Text()
self.parent_conn, child_conn = multiprocessing.Pipe()
self.process = multiprocessing.Process(target=do_something, args=(child_conn,))
def start(self):
self.get_status_updates()
self.process.start()
self.root.mainloop()
def get_status_updates()
status = self.check_pipe()
if status:
self.textbox.add_text(status)
self.root.after(500, self.get_status_updates) # loop every 500ms
def check_pipe():
if self.parent_conn.poll():
status = self.parent_conn.recv()
return status
return None

How should I get my Tkinter IRC client to continuously read data from the IRC server?

I'm writing a little IRC client in python as an exercise. I have a Tkinter.Tk subclass called Main managing the whole application, which creates a socket in its __init__ method. I've played around with sockets in the interactive mode, so I know how to talk to the IRC server with something like this:
>>> s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
>>> s.connect(("irc.foonetic.net", 6667))
>>> s.recv(1000)
":anchor.foonetic.net NOTICE AUTH :*** Looking up your hostname...\r\n:anchor.foonetic.net NOTICE AUTH :*** Couldn't resolve your hostname; using your IP address instead\r\n"
>>> s.send("PASS mypassword\r\n")
That is, I carry on the whole conversation using .send and .recv. Thus to get user input in my Tkinter app, I imagine I'll have an event handler mapped to the Enter key which will call .send. But where do I put the calls to .recv? The only thing I know how to do would be to use a timer to call .recv every few seconds, but that's obviously not a good solution for several reasons. How do I deal with the fact that .recv blocks for several seconds (determined by whatever timeout you set) if there's no data to receive? I realize I could just google "multithreading", but I'd like some guidance on what the best approach is for this specific situation.

In my project, I setup a new thread for long term I/O like socket read/write. To write a practical GUI program, you have to face multithread soon or later. That's because GUI framework has an event queue, and an event loop. The event loop is typically a while loop, in which it get events from event queue and dispatch this events to registered functions. Like the following:
while event is not QUIT:
event = event_queue.get(block=True)
dispatch(event)
In dispatch, all callback functions registered on that event is called directly.
Such code works in the GUI thread, and if you do long term I/O or blocking action in a GUI callback, the thread is blocked in that callback. In terms of event loop, the program is blocked in the dispatch function which called the blocked callback function. Any new event in the event queue will not be processed. As a result, the program looks like dead because the updating event of GUI is blocked.
When you have setup a worker thread to handle time consuming things, don't try to operate GUI widgets directly from that worker thread. Most GUI frameworks are not thread safe, they keep operation sequence by the event queue. And operating a widget in non-GUI threads will break this sequence.
We can add event to event queue from non-GUI thread, and let GUI thread handle that event, to keep the sequence. This is the normal way for some common language, but not for python. In python, function and method are first class object, so we can put then in the queue. Unfortunately, the event queue for tkinter does not support this feature.
In Programming Python by Mark Lutz there is great cover of tkinter programming. In this book, the author introduced a great method to do multithread in tkinter. Here is my demo:
# python3 source code
from tkinter import *
from tkinter.ttk import *
import threading
import time
import queue
root = Tk()
msg = StringVar()
Label(root, textvariable=msg).pack()
# This is our own event queue, each element should be in this form:
# (function_to_be_called_from_gui_thread, function_arguments)
# In python, functions are objects and can be put in a queue.
my_event_queue = queue.Queue()
def worker():
"""
This is a time consuming worker, it takes 1 second for each task.
If you put such a worker in the GUI thread, the GUI will be blocked.
"""
task_counter = 0
while True:
time.sleep(1) # simulate a time consuming task
# show how many tasks finished in the Label. We put this action in my_event_queue instead of handle
# it from this worker thread which is not safe. This action will be handled by my_event_handler which is
# called from GUI thread.
my_event_queue.put((msg.set, '{} tasks finished.'.format(task_counter)))
task_counter += 1
def my_event_handler():
"""
Query my_event_queue, and handle one event per time.
"""
try:
func, *args = my_event_queue.get(block=False)
except queue.Empty:
pass
else:
func(*args)
# At last schedule handling for next time.
# Every 100 ms, my_event_handler will be called
root.after(100, my_event_handler)
threading.Thread(target=worker, daemon=True).start() # start worker in new thread
my_event_handler() # start handler, after root.mainloop(), this method will be called every 100ms. Or you can use root.after(100, my_event_handler)
root.mainloop()
Here is the running picture. You can see I adjust the window size when it is running.(Well I have not enough reputation to post images, so you have to try it yourself)
At last I would suggest you to take a look at Programming Python for tkinter programming.
All Python code are in Python3.

I'm pretty new to Python in general and very new to Tk/ttk. But here's an example of what I've been playing with for event triggering/signaling and worker thread stuff in Tk/ttk. I know some people will hate the singleton decorator and I know there are other ways to call code from other classes but the trigger class is very convenient and the worker class works like a charm. Together they make things super easy.
Credits:
The worker class is a very slightly modified version of the GObject worker found in Pithos and the singleton decorator is a very slightly modified version of something I found here on stackoverflow somewhere.
import sys
import tkinter
from tkinter import ttk
from tkinter import StringVar
import threading
import queue
import traceback
import time
class TkWorkerThreadDemo:
def __init__(self):
self.root = tkinter.Tk()
self.trigger = Trigger.Singleton()
self.trigger.connect_event('enter_main_thread', self.enter_main_thread)
self.worker = Worker()
self.root.title('Worker Thread Demo')
self.root.resizable(width='False', height='False')
self.test_label_text = StringVar()
self.test_label_text.set('')
self.slider_label_text = StringVar()
self.slider_label_text.set('Press either button and try to move the slider around...')
mainframe = ttk.Frame(self.root)
test_label = ttk.Label(mainframe, anchor='center', justify='center', textvariable=self.test_label_text)
test_label.pack(padx=8, pady=8, fill='x')
slider_label = ttk.Label(mainframe, anchor='center', justify='center', textvariable=self.slider_label_text)
slider_label.pack(padx=8, pady=8, expand=True, fill='x')
self.vol_slider = ttk.Scale(mainframe, from_=0, to=100, orient='horizontal', value='100', command=self.change_slider_text)
self.vol_slider.pack(padx=8, pady=8, expand=True, fill='x')
test_button = ttk.Button(mainframe, text='Start Test with a Worker Thread', command=self.with_worker_thread)
test_button.pack(padx=8, pady=8)
test_button = ttk.Button(mainframe, text='Start Test in the Main Thread', command=self.without_worker_thread)
test_button.pack(padx=8, pady=8)
mainframe.pack(padx=8, pady=8, expand=True, fill='both')
self.root.geometry('{}x{}'.format(512, 256))
def enter_main_thread(self, callback, result):
self.root.after_idle(callback, result)
def in_a_worker_thread(self):
msg = 'Hello from the worker thread!!!'
time.sleep(10)
return msg
def in_a_worker_thread_2(self, msg):
self.test_label_text.set(msg)
def with_worker_thread(self):
self.test_label_text.set('Waiting on a message from the worker thread...')
self.worker.send(self.in_a_worker_thread, (), self.in_a_worker_thread_2)
def in_the_main_thread(self):
msg = 'Hello from the main thread!!!'
time.sleep(10)
self.in_the_main_thread_2(msg)
def in_the_main_thread_2(self, msg):
self.test_label_text.set(msg)
def without_worker_thread(self):
self.test_label_text.set('Waiting on a message from the main thread...')
self.root.update_idletasks()#without this the text wil not get set?
self.in_the_main_thread()
def change_slider_text(self, slider_value):
self.slider_label_text.set('Slider value: %s' %round(float(slider_value)))
class Worker:
def __init__(self):
self.trigger = Trigger.Singleton()
self.thread = threading.Thread(target=self._run)
self.thread.daemon = True
self.queue = queue.Queue()
self.thread.start()
def _run(self):
while True:
command, args, callback, errorback = self.queue.get()
try:
result = command(*args)
if callback:
self.trigger.event('enter_main_thread', callback, result)
except Exception as e:
e.traceback = traceback.format_exc()
if errorback:
self.trigger.event('enter_main_thread', errorback, e)
def send(self, command, args=(), callback=None, errorback=None):
if errorback is None: errorback = self._default_errorback
self.queue.put((command, args, callback, errorback))
def _default_errorback(self, error):
print("Unhandled exception in worker thread:\n{}".format(error.traceback))
class singleton:
def __init__(self, decorated):
self._decorated = decorated
self._instance = None
def Singleton(self):
if self._instance:
return self._instance
else:
self._instance = self._decorated()
return self._instance
def __call__(self):
raise TypeError('Singletons must be accessed through `Singleton()`.')
#singleton
class Trigger:
def __init__(self):
self._events = {}
def connect_event(self, event_name, func, *args, **kwargs):
self._events[event_name] = func
def disconnect_event(self, event_name, *args, **kwargs):
if event_name in self._events:
del self._events[event_name]
def event(self, event_name, *args, **kwargs):
if event_name in self._events:
return self._events[event_name](*args, **kwargs)
def main():
demo = TkWorkerThreadDemo()
demo.root.mainloop()
sys.exit(0)
if __name__ == '__main__':
main()

PyQT and threads

I am developing an application that uses multiple threads to gather data from a variety of network devices. I'm using PyQT to display the collected data on a GUI. I am using regular python threads (from thread, threading) in my app (instead of QThread). In order to update the GUI on the different threads, I use a lock (thread.allocate_lock()). So, anytime a GUI update will happen, I call with lock, update GUI. Any concerns about this?

I'm pretty sure that updating the GUI from different threads is dangerous in Qt, even if you try to lock things in your own code. For one thing, Qt might be doing its own event processing on the main thread, and it will not acquire your lock to protect objects that it might modify. On this page in the Qt docs, the fact that QWidget is not reentrant or thread-safe is explicitly mentioned.
I recommend that you post the collected data, or a processed version of it, back to the main thread. Use a queued signal/slot connection, or a custom QEvent and QApplication::postEvent to do this. In the previous question that jkerian mentions, it says that you'll have to use QThread instead of python's threads if you want event posting to work correctly.

This is a late reply but I wanted to share what I found. This is code from WickedDevice Blog that I found useful to understand threads and PyQt:
#authors: Dirk Swart, Doudewijn Rempt, Jacob Hallen
import sys, time, threading, random, Queue
from PyQt4 import QtGui, QtCore as qt
import serial
SERIALPORT = 'COM6'
class GuiPart(QtGui.QMainWindow):
def __init__(self, queue, endcommand, *args):
QtGui.QMainWindow.__init__(self, *args)
self.setWindowTitle('Arduino Serial Demo')
self.queue = queue
# We show the result of the thread in the gui, instead of the console
self.editor = QtGui.QTextEdit(self)
self.setCentralWidget(self.editor)
self.endcommand = endcommand
def closeEvent(self, ev):
self.endcommand()
def processIncoming(self):
"""
Handle all the messages currently in the queue (if any).
"""
while self.queue.qsize():
try:
msg = self.queue.get(0)
# Check contents of message and do what it says
# As a test, we simply print it
self.editor.insertPlainText(str(msg))
except Queue.Empty:
pass
class ThreadedClient:
"""
Launch the main part of the GUI and the worker thread. periodicCall and
endApplication could reside in the GUI part, but putting them here
means that you have all the thread controls in a single place.
"""
def __init__(self):
# Create the queue
self.queue = Queue.Queue()
# Set up the GUI part
self.gui=GuiPart(self.queue, self.endApplication)
self.gui.show()
# A timer to periodically call periodicCall :-)
self.timer = qt.QTimer()
qt.QObject.connect(self.timer,
qt.SIGNAL("timeout()"),
self.periodicCall)
# Start the timer -- this replaces the initial call to periodicCall
self.timer.start(100)
# Set up the thread to do asynchronous I/O
# More can be made if necessary
self.running = 1
self.thread1 = threading.Thread(target=self.workerThread1)
self.thread1.start()
def periodicCall(self):
"""
Check every 100 ms if there is something new in the queue.
"""
self.gui.processIncoming()
if not self.running:
root.quit()
def endApplication(self):
self.running = 0
def workerThread1(self):
"""
This is where we handle the asynchronous I/O.
Put your stuff here.
"""
while self.running:
#This is where we poll the Serial port.
#time.sleep(rand.random() * 0.3)
#msg = rand.random()
#self.queue.put(msg)
ser = serial.Serial(SERIALPORT, 115200)
msg = ser.readline();
if (msg):
self.queue.put(msg)
else: pass
ser.close()
if __name__ == "__main__":
#rand = random.Random()
root = QtGui.QApplication(sys.argv)
client = ThreadedClient()
sys.exit(app.exec_())

I use pyqtSignal and Python's threading. You can create threads and when the thread is completed have it send a signal to update your GUI.