I'm building an interface on top of some analysis code I've written that executes some SQL and processes the query results. There's logging surrounding a number of the events in this analysis code that I would like to expose to the user. Because the analysis code is rather long-running, and because I don't want the UI to block, thus far I've done this through putting the analysis function in to its own thread.
Simplified example of what I have now (complete script):
import sys
import time
import logging
from PySide2 import QtCore, QtWidgets
def long_task():
logging.info('Starting long task')
time.sleep(3) # this would be replaced with a real task
logging.info('Long task complete')
class LogEmitter(QtCore.QObject):
sigLog = QtCore.Signal(str)
class LogHandler(logging.Handler):
def __init__(self):
super().__init__()
self.emitter = LogEmitter()
def emit(self, record):
msg = self.format(record)
self.emitter.sigLog.emit(msg)
class LogDialog(QtWidgets.QDialog):
def __init__(self, parent=None):
super().__init__(parent)
log_txt = QtWidgets.QPlainTextEdit(self)
log_txt.setReadOnly(True)
layout = QtWidgets.QHBoxLayout(self)
layout.addWidget(log_txt)
self.setWindowTitle('Event Log')
handler = LogHandler()
handler.emitter.sigLog.connect(log_txt.appendPlainText)
logger = logging.getLogger()
logger.addHandler(handler)
logger.setLevel(logging.INFO)
class Worker(QtCore.QThread):
results = QtCore.Signal(object)
def __init__(self, func, *args, **kwargs):
super().__init__()
self.func = func
self.args = args
self.kwargs = kwargs
def run(self):
results = self.func(*self.args, **self.kwargs)
self.results.emit(results)
class MainWindow(QtWidgets.QMainWindow):
def __init__(self):
super().__init__()
widget = QtWidgets.QWidget()
layout = QtWidgets.QHBoxLayout(widget)
start_btn = QtWidgets.QPushButton('Start')
start_btn.clicked.connect(self.start)
layout.addWidget(start_btn)
self.setCentralWidget(widget)
self.log_dialog = LogDialog()
self.worker = None
def start(self):
if not self.worker:
self.log_dialog.show()
logging.info('Run Starting')
self.worker = Worker(long_task)
self.worker.results.connect(self.handle_result)
self.worker.start()
def handle_result(self, result=None):
logging.info('Result received')
self.worker = None
if __name__ == '__main__':
app = QtWidgets.QApplication()
win = MainWindow()
win.show()
sys.exit(app.exec_())
This works fine, except that I need to be able to allow the user to stop the execution of the analysis code. Everything I've read indicates that there is no way to interrupt threads nicely, so using the multiprocessing library seems to be the way to go (there's no way to re-write the analysis code to allow for periodic polling, since the majority of time is spent just waiting for the queries to return results). It's easy enough to get the same functionality in terms of executing the analysis code in a way that doesn't block the UI by using multiprocessing.Pool and apply_async.
E.g. replacing MainWindow from above with:
class MainWindow(QtWidgets.QMainWindow):
def __init__(self):
super().__init__()
widget = QtWidgets.QWidget()
layout = QtWidgets.QHBoxLayout(widget)
start_btn = QtWidgets.QPushButton('Start')
start_btn.clicked.connect(self.start)
layout.addWidget(start_btn)
self.setCentralWidget(widget)
self.log_dialog = LogDialog()
self.pool = multiprocessing.Pool()
self.running = False
def start(self):
if not self.running:
self.log_dialog.show()
logging.info('Run Starting')
self.pool.apply_async(long_task, callback=self.handle_result)
def handle_result(self, result=None):
logging.info('Result received')
self.running = False
But I can't seem to figure out how I would go about retrieving the logging output from the child process and passing it to the parent to update the log dialog. I've read through just about every SO question on this as well as the cookbook examples of how to handle writing to a single log file from multiple processes, but I can't wrap my head around how to adapt those ideas to what I'm trying to do here.
Edit
So trying to figure out what might be going on for why I'm seeing different behavior than #eyllanesc I added:
logger = logging.getLogger()
print(f'In Func: {logger} at {id(logger)}')
and
logger = logging.getLogger()
print(f'In Main: {logger} at {id(logger)}')
to long_task and Mainwindow.start, respectively. When I run main.py I get:
In Main: <RootLogger root (INFO)> at 2716746681984
In Func: <RootLogger root (WARNING)> at 1918342302352
which seems to be what was described in this SO question
This idea of using a Queue and QueueHandler though as a solution seems similar to #eyllanesc's original solution
The signals do not transmit data between processes, so for this case a Pipe must be used and then emit the signal:
# other imports
import threading
# ...
class LogHandler(logging.Handler):
def __init__(self):
super().__init__()
self.r, self.w = multiprocessing.Pipe()
self.emitter = LogEmitter()
threading.Thread(target=self.listen, daemon=True).start()
def emit(self, record):
msg = self.format(record)
self.w.send(msg)
def listen(self):
while True:
try:
msg = self.r.recv()
self.emitter.sigLog.emit(msg)
except EOFError:
break
# ...
In case anyone wanders in to this down the road, using QueueHandler and QueueListener leads to a solution that works on Windows as well. Borrowed heavily from this answer to a similar question:
import logging
import sys
import time
import multiprocessing
from logging.handlers import QueueHandler, QueueListener
from PySide2 import QtWidgets, QtCore
def long_task():
logging.info('Starting long task')
time.sleep(3) # this would be replaced with a real task
logging.info('Long task complete')
def worker_init(q):
qh = QueueHandler(q)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
logger.addHandler(qh)
class LogEmitter(QtCore.QObject):
sigLog = QtCore.Signal(str)
class LogHandler(logging.Handler):
def __init__(self):
super().__init__()
self.emitter = LogEmitter()
def emit(self, record):
msg = self.format(record)
self.emitter.sigLog.emit(msg)
class LogDialog(QtWidgets.QDialog):
def __init__(self, parent=None):
super().__init__(parent)
self.log_txt = QtWidgets.QPlainTextEdit(self)
self.log_txt.setReadOnly(True)
layout = QtWidgets.QHBoxLayout(self)
layout.addWidget(self.log_txt)
self.setWindowTitle('Event Log')
class MainWindow(QtWidgets.QMainWindow):
def __init__(self):
super().__init__()
widget = QtWidgets.QWidget()
layout = QtWidgets.QHBoxLayout(widget)
start_btn = QtWidgets.QPushButton('Start')
start_btn.clicked.connect(self.start)
layout.addWidget(start_btn)
self.setCentralWidget(widget)
self.log_dialog = LogDialog()
self.running = False
# sets up handler that will be used by QueueListener
# which will update the LogDialoag
handler = LogHandler()
handler.emitter.sigLog.connect(self.log_dialog.log_txt.appendPlainText)
self.q = multiprocessing.Queue()
self.ql = QueueListener(self.q, handler)
self.ql.start()
# main process should also log to a QueueHandler
self.main_log = logging.getLogger('main')
self.main_log.propagate = False
self.main_log.setLevel(logging.INFO)
self.main_log.addHandler(QueueHandler(self.q))
self.pool = multiprocessing.Pool(1, worker_init, [self.q])
def start(self):
if not self.running:
self.log_dialog.show()
self.main_log.info('Run Starting')
self.pool.apply_async(long_task, callback=self.handle_result)
def handle_result(self, result=None):
time.sleep(2)
self.main_log.info('Result received')
self.running = False
def closeEvent(self, _):
self.ql.stop()
if __name__ == '__main__':
app = QtWidgets.QApplication()
win = MainWindow()
win.show()
sys.exit(app.exec_())
Related
I want to use multiple imported function with arguments that takes some while to run. I want a 'working' progress bar that track the processes of that function. I have followed 2 questions already here.
Connect an imported function to Qt5 progress bar without dependencies
Report progress to QProgressBar using variable from an imported module
The difference is that the thread can take any function which can have arguments. The function also not needs to yield the percent to return to the progressbar. The progressbar always start at 0%.
I copied a snippet from first link and modified it for example purpose.
from external_script import long_running_function
class Actions(QDialog):
def __init__(self):
super().__init__()
self.initUI()
def initUI(self):
self.setWindowTitle('Progress Bar')
self.progress = QProgressBar(self)
self.button = QPushButton('Start', self)
self.show()
self.button.clicked.connect(self.onButtonClick)
def onButtonClick(self):
long_running_function(**kwargs) # This can be any function that takes argument/s
self.progress.setValue(value)
Do not get too complicated with the answers as they are limited to a very particular context. In general the logic is to pass a QObject to it that updates the percentage value and then emits a signal with that value. For example a simple solution is to use the threading module:
import sys
import threading
from PyQt5 import QtCore, QtWidgets
class PercentageWorker(QtCore.QObject):
started = QtCore.pyqtSignal()
finished = QtCore.pyqtSignal()
percentageChanged = QtCore.pyqtSignal(int)
def __init__(self, parent=None):
super().__init__(parent)
self._percentage = 0
#property
def percentage(self):
return self._percentage
#percentage.setter
def percentage(self, value):
if self._percentage == value:
return
self._percentage = value
self.percentageChanged.emit(self.percentage)
def start(self):
self.started.emit()
def finish(self):
self.finished.emit()
class FakeWorker:
def start(self):
pass
def finish(self):
pass
#property
def percentage(self):
return 0
#percentage.setter
def percentage(self, value):
pass
import time
def long_running_function(foo, baz="1", worker=None):
if worker is None:
worker = FakeWorker()
worker.start()
while worker.percentage < 100:
worker.percentage += 1
print(foo, baz)
time.sleep(1)
worker.finish()
class Widget(QtWidgets.QWidget):
def __init__(self, parent=None):
super().__init__(parent)
self.progress = QtWidgets.QProgressBar()
self.button = QtWidgets.QPushButton("Start")
lay = QtWidgets.QVBoxLayout(self)
lay.addWidget(self.button)
lay.addWidget(self.progress)
self.button.clicked.connect(self.launch)
def launch(self):
worker = PercentageWorker()
worker.percentageChanged.connect(self.progress.setValue)
threading.Thread(
target=long_running_function,
args=("foo",),
kwargs=dict(baz="baz", worker=worker),
daemon=True,
).start()
if __name__ == "__main__":
app = QtWidgets.QApplication(sys.argv)
w = Widget()
w.show()
sys.exit(app.exec_())
The simple idea is that user inputs duration in seconds, and presses a PyQt button, that calls a function that creates a python subprocess and runs windump via it. Then time sleep is used to wait for user defined duration and then process.terminate(), terminates it (code below)
def windump_exec(duration):
p = s.Popen(['windump', '-i', '3', '-w', 'packets.pcap'], stdout=s.PIPE)
time.sleep(duration)
p.terminate()
Now once this is done, scapy reads .pcap file and I show stuff on the screen in short. While this is happening QWaitingSpinner is running, and to handle this I run the above logic (including scapy) using QRunnable (code below)
class ThreadRunnable(QRunnable):
def __init__(self, _time, filler):
QRunnable.__init__(self)
self.time = _time
self.filler = filler
self.signal = RunnableSignal()
def run(self):
windump_exec(self.time)
packets = parse_data()
self.filler(packets)
self.signal.result.emit()
The Problem is that the windump code works fine on it's own, but inside the QThread it doesn't create an output file and hence scapy has nothing to read (open), and it gives error.
Instead of using Popen with QThread you can use QProcess, in my test I have used tcpdump but I suppose that changing to windump should have the same behavior:
import os
from PyQt5 import QtCore, QtGui, QtWidgets
from scapy.all import rdpcap
import psutil
CURRENT_DIR = os.path.dirname(os.path.realpath(__file__))
class DumpProcesor(QtCore.QObject):
started = QtCore.pyqtSignal()
finished = QtCore.pyqtSignal()
def __init__(self, parent=None):
super().__init__(parent)
self._process = QtCore.QProcess()
self._timer = QtCore.QTimer(singleShot=True)
self._timer.timeout.connect(self.handle_timeout)
self._pid = -1
#property
def process(self):
return self._process
#property
def timer(self):
return self._timer
#QtCore.pyqtSlot()
def start(self):
self.started.emit()
status, self._pid = self._process.startDetached()
if status:
self._timer.start()
else:
self.finished.emit()
#QtCore.pyqtSlot()
def handle_timeout(self):
if self._pid > 0:
p = psutil.Process(self._pid)
p.terminate()
QtCore.QTimer.singleShot(100, self.finished.emit)
class Widget(QtWidgets.QWidget):
def __init__(self, parent=None):
super().__init__(parent)
self.log_te = QtWidgets.QTextEdit(readOnly=True)
self.time_sb = QtWidgets.QSpinBox(minimum=1)
self.start_btn = QtWidgets.QPushButton(self.tr("Start"))
grid_layout = QtWidgets.QGridLayout(self)
grid_layout.addWidget(self.log_te, 0, 0, 1, 3)
grid_layout.addWidget(QtWidgets.QLabel("Time (seg):"), 1, 0)
grid_layout.addWidget(self.time_sb, 1, 1)
grid_layout.addWidget(self.start_btn, 1, 2)
self.dump_procesor = DumpProcesor(self)
self.dump_procesor.process.setProgram("tcpdump")
filename = os.path.join(CURRENT_DIR, "packets.pcap")
self.dump_procesor.process.setArguments(["-i", "3", "-w", filename])
self.start_btn.clicked.connect(self.start)
self.dump_procesor.finished.connect(self.on_finished)
#QtCore.pyqtSlot()
def start(self):
self.log_te.clear()
self.start_btn.setDisabled(True)
self.dump_procesor.timer.setInterval(self.time_sb.value() * 1000)
self.dump_procesor.start()
#QtCore.pyqtSlot()
def on_finished(self):
self.start_btn.setDisabled(False)
filename = os.path.join(CURRENT_DIR, "packets.pcap")
packets = rdpcap(filename)
for packet in packets:
t = packet.show(dump=True)
self.log_te.append(t)
def main():
import sys
app = QtWidgets.QApplication(sys.argv)
w = Widget()
w.show()
sys.exit(app.exec_())
if __name__ == "__main__":
main()
I've constructed a window in PyQt 5, which by clicking on the "optimize" button, the program reads the "Gurobi-model.lp" file (click here to get the file), and optimizes it by the help of the Gurobi software. How can I display the logs of the Gurobi on a QTextBrowser?
I found some functions in the Gurobi such as OutputFlag, LogFile, and LogToConsole. Might these functions be helpful not?
For those who are not familiar with Gurobi, the Gurobi optimizer uses Python as an interface, and produces some logs that allows you to track the progress of the optimization. These logs are printed in the console during the optimization, and somehow, responding my question doesn't need to know anything about the Gurobi.
In the below code, I've found a way to show the logs in the QTextBrowser, but the logs are represented when the optimization process is completely done. I want the logs to be represented exactly during the optimization process.
import sys
from PyQt5.QtWidgets import *
from gurobipy import *
from io import *
class MyWindow(QWidget):
def __init__(self):
QWidget.__init__(self)
self.pb = QPushButton(self.tr("optimize"))
self.log_text = QTextBrowser()
layout = QVBoxLayout(self)
layout.addWidget(self.pb)
layout.addWidget(self.log_text)
self.setLayout(layout)
self.pb.clicked.connect(self.optimize)
def optimize(self):
f = StringIO()
sys.stdout = StringIO()
self.m = read('Gurobi-model.lp')
self.m.optimize()
self.log_text.append(sys.stdout.getvalue() )
def main():
app = QApplication(sys.argv)
w = MyWindow()
w.show()
sys.exit(app.exec_())
if __name__ == "__main__":
main()
The task of optimizing is heavy, so it should not be executed in the same thread of the GUI, nor in the same process. For this you should use the multiprocessing module. On the other hand, if you need to show the output of the console in the QTextBrowser, you must use the logging module, passing it through signal (for the last part, use the answer of this post).
import sys
import logging
import multiprocessing
from logging.handlers import QueueHandler, QueueListener
from PyQt5 import QtCore, QtWidgets
from gurobipy import *
class LogEmitter(QtCore.QObject):
sigLog = QtCore.pyqtSignal(str)
class LogHandler(logging.Handler):
def __init__(self):
super().__init__()
self.emitter = LogEmitter()
def emit(self, record):
msg = self.format(record)
self.emitter.sigLog.emit(msg)
def long_task():
m = read('Gurobi-model.lp')
m.optimize()
def worker_init(q):
qh = QueueHandler(q)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
logger.addHandler(qh)
class MyWindow(QtWidgets.QWidget):
def __init__(self, parent=None):
super(MyWindow, self).__init__(parent)
self.pb = QtWidgets.QPushButton(self.tr("optimize"),
clicked=self.start_optimize)
self.log_text = QtWidgets.QPlainTextEdit(readOnly=True)
layout = QtWidgets.QVBoxLayout(self)
layout.addWidget(self.pb)
layout.addWidget(self.log_text)
self.running = False
handler = LogHandler()
handler.emitter.sigLog.connect(self.log_text.appendPlainText)
self.q = multiprocessing.Queue()
self.ql = QueueListener(self.q, handler)
self.ql.start()
self.main_log = logging.getLogger('main')
self.main_log.propagate = False
self.main_log.setLevel(logging.INFO)
self.main_log.addHandler(QueueHandler(self.q))
self.pool = multiprocessing.Pool(1, worker_init, [self.q])
#QtCore.pyqtSlot()
def start_optimize(self):
if not self.running:
self.pool.apply_async(long_task, callback=self.handle_result)
def handle_result(self, result=None):
self.running = False
def closeEvent(self, event):
self.ql.stop()
super(MyWindow, self).closeEvent(event)
def main():
app = QtWidgets.QApplication(sys.argv)
w = MyWindow()
w.show()
sys.exit(app.exec_())
if __name__ == "__main__":
main()
I am trying to create and run a new thread each time I want to pop a message box from main thread. Wondering if this code below will cause memory leak or even a good practice. If it can be improved, how to?
class UIThread (threading.Thread):
def __init__(self, message):
print "UIThread init(). msg = " + message
this_thread = threading.Thread.__init__(self)
this_thread.daemon = True
self.message = message
self.qtapp = QApplication(sys.argv)
self.w = QWidget()
def run(self):
print "UIThread running"
result = QMessageBox.warning(self.w, 'WARNING', self.message, QMessageBox.Yes | QMessageBox.No, QMessageBox.No)
if result == QMessageBox.Yes:
print 'Yes.'
else:
print 'No.'
sys.exit(self.qtapp.exec_())
...
messagebox = UIThread("Test")
messagebox.start()
....
messagebox = UIThread("Test 2")
messagebox.start()
....
# will pop up more message boxes throughout the program
No it's not good practice. You should only do GUI stuff, including showing message boxes, from the main thread. Use signals and slots to communicate with the main thread, which can show the message box and provide a response for your secondary thread.
Here's a simple example where a worker thread signals the main thread to show a message box, then checks the responses dictionary for the message box response (basic dictionary operations are thread safe). This example should also work for multiple threads as the responses are keyed under the thread name.
from PyQt4 import QtCore, QtGui
import threading, time, sys
class Worker(QtCore.QObject):
def __init__(self, app):
super(Worker, self).__init__()
self.app = app
def do_stuff(self):
thread_name = threading.current_thread().name
self.app.responses[thread_name] = None
self.app.showMessageBox.emit(thread_name,
'information',
'Hello',
'Thread {} sent this message.'.format(thread_name))
while self.app.responses[thread_name] is None:
time.sleep(0.1)
print 'Thread {} got response from message box: {}'.format(thread_name, self.app.responses[thread_name])
class MainWindow(QtGui.QMainWindow):
showMessageBox = QtCore.pyqtSignal(str, str, str, str)
def __init__(self, sys_argv):
super(MainWindow, self).__init__(sys_argv)
self.responses = {}
self.showMessageBox.connect(self.on_show_message_box)
self.worker = Worker(self)
self.thread = QtCore.QThread()
self.worker.moveToThread(self.thread)
self.thread.started.connect(self.worker.do_stuff)
self.thread.start()
def on_show_message_box(self, id, severity, title, text):
self.responses[str(id)] = getattr(QtGui.QMessageBox, str(severity))(self, title, text)
if __name__ == '__main__':
app = QtGui.QApplication(sys.argv)
win = MainWindow(None)
win.show()
sys.exit(app.exec_())
I'm trying to update the text in a Qt GUI object via a QThread in PyQt but I just get the error QPixmap: It is not safe to use pixmaps outside the GUI thread, then it crashes. I would really appreciate any help, thanks.
class MainWindow(QMainWindow, Ui_MainWindow):
def __init__(self, parent = None):
QMainWindow.__init__(self, parent)
self.setupUi(self)
self.output = Output()
def __del__ (self):
self.ui = None
#pyqtSignature("")
def on_goBtn_released(self):
threadnum = 1
#start threads
for x in xrange(threadnum):
thread = TheThread()
thread.start()
class Output(QWidget, Ui_Output):
def __init__(self, parent = None):
QWidget.__init__(self, parent)
self.setupUi(self)
self.ui = Ui_Output
self.show()
def main(self):
self.textBrowser.append("sdgsdgsgsg dsgdsg dsgds gsdf")
class TheThread(QtCore.QThread):
trigger = pyqtSignal()
def __init__(self):
QtCore.QThread.__init__(self)
def __del__(self):
self.wait()
def run(self):
self.trigger.connect(Output().main())
self.trigger.emit()
self.trigger.connect(Output().main())
This line is problematic. You are instantiating a class in the thread which looks like a widget. This is wrong. You shouldn't use GUI elements in a different thread. All GUI related code should run in the same thread with the event loop.
The above line is also wrong in terms of design. You emit a custom signal from your thread and this is a good way. But the object to process this signal should be the one that owns/creates the thread, namely your MainWindow
You also don't keep a reference to your thread instance. You create it in a method, but it is local. So it'll be garbage collected, you probably would see a warning that it is deleted before it is finished.
Here is a minimal working example:
import sys
from PyQt4 import QtGui, QtCore
import time
import random
class MyThread(QtCore.QThread):
trigger = QtCore.pyqtSignal(int)
def __init__(self, parent=None):
super(MyThread, self).__init__(parent)
def setup(self, thread_no):
self.thread_no = thread_no
def run(self):
time.sleep(random.random()*5) # random sleep to imitate working
self.trigger.emit(self.thread_no)
class Main(QtGui.QMainWindow):
def __init__(self, parent=None):
super(Main, self).__init__(parent)
self.text_area = QtGui.QTextBrowser()
self.thread_button = QtGui.QPushButton('Start threads')
self.thread_button.clicked.connect(self.start_threads)
central_widget = QtGui.QWidget()
central_layout = QtGui.QHBoxLayout()
central_layout.addWidget(self.text_area)
central_layout.addWidget(self.thread_button)
central_widget.setLayout(central_layout)
self.setCentralWidget(central_widget)
def start_threads(self):
self.threads = [] # this will keep a reference to threads
for i in range(10):
thread = MyThread(self) # create a thread
thread.trigger.connect(self.update_text) # connect to it's signal
thread.setup(i) # just setting up a parameter
thread.start() # start the thread
self.threads.append(thread) # keep a reference
def update_text(self, thread_no):
self.text_area.append('thread # %d finished' % thread_no)
if __name__ == '__main__':
app = QtGui.QApplication(sys.argv)
mainwindow = Main()
mainwindow.show()
sys.exit(app.exec_())