Develop Reference

How to postpone execution of the QRunnable until signal from previous is emitted

In order not to freeze the PyQT GUI I am using QRunnable (functions to be executed in the background) and QThreadPool. I have three functions out of which one requires results from previous two to run and I am struggling to ensure that the execution of third function starts after two previous return their respective results.
The class for QThreadPool is defined in following way:
class Worker(QtCore.QRunnable):
'''
Worker thread
Inherits from QRunnable to handler worker thread setup, signals and wrap-up.
:param callback: The function callback to run on this worker thread. Supplied args and
kwargs will be passed through to the runner.
:type callback: function
:param args: Arguments to pass to the callback function
:param kwargs: Keywords to pass to the callback function
'''
def __init__(self, fn, *args, **kwargs):
super(Worker, self).__init__()
# Store constructor arguments (re-used for processing)
self.running = None
self.fn = fn
self.args = args
self.kwargs = kwargs
self.signals = WorkerSignals()
# Add the callback to our kwargs
self.kwargs['callback_progress'] = self.signals.progress
self.kwargs['callback_data'] = self.signals.data
# #Slot()
def run(self):
'''
Initialise the runner function with passed args, kwargs.
'''
# Retrieve args/kwargs here; and fire processing using them
try:
self.signals.started.emit()
self.result = self.fn(
*self.args,
**self.kwargs
)
except:
traceback.print_exc()
exctype, value = sys.exc_info()[:2]
self.signals.error.emit((exctype, value, traceback.format_exc()))
else:
self.signals.result.emit(self.result) # Return the result of the processing
finally:
self.signals.finished.emit() # Done
and signals
class WorkerSignals(QtCore.QObject):
'''
Defines the signals available from a running worker thread.
Supported signals are:
finished
No data
error
`tuple` (exctype, value, traceback.format_exc() )
result
`object` data returned from processing, anything
progress
`int` indicating % progress
'''
error = QtCore.Signal(tuple)
started = QtCore.Signal()
finished = QtCore.Signal()
progress = QtCore.Signal(int)
result = QtCore.Signal(object)
data = QtCore.Signal(dict)
the function to be executed is invoked by the following function
def exe_worker_run(self, WorkerPool, function, arguments):
Worker = thread.Worker(function, arguments)
Worker.signals.started.connect(self.sig_thread_start)
Worker.signals.error.connect(self.sig_thread_error)
Worker.signals.result.connect(self.sig_thread_result)
Worker.signals.finished.connect(self.sig_thread_finish)
WorkerPool.start(Worker)
the signal that emits the result is connected to function
def sig_thread_result(self, result):
for key in result.keys():
try:
dfrm = getattr(self, key)
print('{} {} loaded!!!'.format(time.time(), key))
except:
pass
The main problem is that the result of each function is emitted after all functions finished the execution. So what I need is the solution which allows to hold the execution of a QRunnable until the result from previous QRunnable are available.

You can organize tasks into queue and execute them passing to worker one by one over signal-slot mechanism. This way you can use results from one computation to schedule next computation.
from PySide2 import QtCore, QtWidgets, QtGui
class Task:
def __init__(self, taskId):
self._taskId = taskId
def taskId(self):
return self._taskId
def execute():
pass
class TaskPlusOne(Task):
def __init__(self, taskId, value):
super().__init__(taskId)
self._value = value
def execute(self):
QtCore.QThread.currentThread().sleep(3)
return self._value + 1
class Worker(QtCore.QObject):
complete = QtCore.Signal(int, object)
def append(self, task):
print("execute", task.taskId())
res = task.execute()
self.complete.emit(task.taskId(), res)
class Window(QtWidgets.QWidget):
task = QtCore.Signal(object)
def __init__(self, parent = None):
super().__init__(parent)
worker = Worker()
self.task.connect(worker.append)
worker.complete.connect(self.onComplete)
thread = QtCore.QThread()
worker.moveToThread(thread)
thread.start()
self._thread = thread
self._worker = worker
self._queue = [TaskPlusOne(0, 0)]
self.executeOne()
def executeOne(self):
queue = self._queue
if len(queue) == 0:
print("task queue is empty")
return
self.task.emit(queue.pop(0))
def executeAll(self):
while len(self._queue) > 0:
self.executeOne()
def onComplete(self, taskId, res):
print("onComplete", taskId)
if res < 2:
print("append task to queue")
self._queue.append(TaskPlusOne(taskId + 1, res))
self.executeOne()
def closeEvent(self, event):
thread = self._thread
thread.quit()
thread.wait()
super().closeEvent(event)
if __name__ == "__main__":
app = QtWidgets.QApplication([])
widget = Window()
widget.show()
app.exec_()

Half of threads randomly not completing and vanishing

So I currently have a PYQT application utilizing a combination of pythons built-in threading, and QT Qthread/worker system to allow for asynchronous activities
The code for generating the threads/workers is process_batch which is contained in a class called Scraper, which is a QObject
def process_batch(self, batch):
for worker in batch:
self.host.threads.append(QThread())
th = self.host.threads[len(self.host.threads)-1]
worker.moveToThread(th)
th.started.connect(worker.process)
qApp.aboutToQuit.connect(th.quit)
worker.finished.connect(worker.deleteLater)
th.finished.connect(th.deleteLater)
for thread in self.host.threads:
thread.start()
self.activeThreads += 1
the the self.host.threads is a list variable helf by the ScraperHost object, the batch variable is a standard python list with 50 of the following class:
class ScraperWorker(QObject):
complete = pyqtSignal(str)
start = pyqtSignal(str)
finished = pyqtSignal()
def __init__(self, url, task, _id):
QObject.__init__(self)
self.url = url
self.task = task
self.id = _id
self.start.connect(self.process)
def __str__(self):
return self.url
#pyqtSlot()
def process(self):
time.sleep(float(random.random() * random.randint(1,3)))
#self.complete.emit(str(self.id))
self.finished.emit()
The time.sleep was added to see if spacing out the thread's completions randomly would fix the issue, it did not
Originally I was running into an issue of the threads getting deleted before finishing when the process_batch function completed, so I implemented the self.host.threads to keep them from getting garbage collected.
threads are contained below in :
class ScraperHost(threading.Thread):
def __init__(self, threadID, urls, task):
threading.Thread.__init__(self)
self.threadID = threadID
self.scraper = Scraper(urls, task, self)
self.threads = []
def run(self):
self.scraper.run()
The scraper host is created in my MainWindow by:
def selectFileList(self):
options = QFileDialog.Options()
fileName, _ = QFileDialog.getOpenFileName(self, "Select List","", "Excel File (*.xls *.xlsx)", options=options)
if fileName:
...
...
self.ScraperHost = ScraperHost(0, urls, "index")
self.ScraperHost.run()
As it is currently I am able to run the function/program with zero errors, but it only completes exactly half of the threads prior to finishing. It gives me no errors, messages, or any indication as to why they aren't finishing. If I increase the batch size to 100, it finishes 50. If I run it in debug mode in VS Code, i can see that all 50 threads are created correctly, and are all in the host. threads variable. If ** I step through the thread's creation 1 by 1 in debug mode they will all finish correctly**, but if I just run the program normally only exactly half will finish. The 25 threads that finish are different every time due to the random time they wait. The self.finished.emit() of the worker just prints the thread's ID, which is how I am seeing how many finish
EDIT: Reproducable Sample: Side Note: Gets through more, sometimes all, but with no consistency, of the threads when ran from idle as opposed to VS Code
from PyQt5.QtWidgets import *
from PyQt5.QtGui import *
from PyQt5.QtCore import *
import threading
import time
import random
scraperhost = None
class ScraperHost(threading.Thread):
def __init__(self, threadID, urls, task):
threading.Thread.__init__(self)
self.threadID = threadID
self.scraper = Scraper(urls, task, self)
self.threads = []
def run(self):
self.scraper.run()
class Scraper(QObject):
output = pyqtSignal(str)
complete = pyqtSignal()
def __init__(self, urls, task, host):
QObject.__init__(self)
self.urls = urls
self.batches = []
self.activeThreads = 0
self.task = task
self.host = host
def __del__(self):
self.wait()
def run(self):
batch_size = 50
batch_count = 1
for batch in range(0, batch_count):
self.batches.append([])
for i in range (0, batch_size):
try:
c_url = "www.website.com"
worker = ScraperWorker(c_url, self.task, i)
worker.complete.connect(self.worker_finished)
self.batches[batch].append(worker)
except IndexError:
break
if len(self.batches) == 0:
return
self.process_batch(self.batches.pop())
#pyqtSlot(str)
def worker_finished(self, ss):
self.activeThreads -= 1
print(self.activeThreads)
def process_batch(self, batch):
for worker in batch:
self.host.threads.append(QThread())
th = self.host.threads[len(self.host.threads)-1]
worker.moveToThread(th)
th.started.connect(worker.process)
qApp.aboutToQuit.connect(th.quit)
worker.finished.connect(worker.deleteLater)
th.finished.connect(th.deleteLater)
for thread in self.host.threads:
thread.start()
self.activeThreads += 1
class ScraperWorker(QObject):
complete = pyqtSignal(str)
start = pyqtSignal(str)
finished = pyqtSignal()
def __init__(self, url, task, _id):
QObject.__init__(self)
self.url = url
self.task = task
self.id = _id
self.start.connect(self.process)
def __str__(self):
return self.url
#pyqtSlot()
def process(self):
time.sleep(float(random.random() * random.randint(1,3)))
self.complete.emit(str(self.id))
self.finished.emit()
def main():
app = QApplication([])
ex = QWidget()
ex.show()
global scraperhost
scraperhost = ScraperHost(0, [], "index")
scraperhost.start()
app.exec_()
return
main()

Program has to wait till two threads (Sql query execution) are completed

I have a program which compares to DB table values and i have created a GUI in PyQt5. I have created two threads one for querying each table and then program has to wait till two threads are completed. My code below
from PySide2 import QtWidgets
from PySide2 import QtGui
from PySide2 import QtCore
from Main_interface import Ui_mainWindow
import pandas as pd
class mainWindow(QtWidgets.QMainWindow, Ui_mainWindow):
sqlClicked1 = QtCore.Signal(str)
sqlClicked2 = QtCore.Signal(str)
def __init__(self, parent=None):
super(mainWindow, self).__init__(parent)
self.setupUi(self)
self.thread = QtCore.QThread(self)
self.thread.start()
self.obj = Worker()
self.obj.moveToThread(self.thread)
self.sqlClicked.connect(self.obj.runsql_MC)
self.sqlClicked1.connect(self.obj.runsql_IRI)
self.obj.error.connect(self.on_error)
def run_report(self):
sqlquery1 = "Select * from table1"
sqlquery2 = "Select * from table2"
df1 = self.sqlClicked1.emit(sqlquery1)
df2 = self.sqlClicked2.emit(sqlquery2)
self.sqlClicked1.finished.connect(self.on_finished)
self.sqlClicked2.finished.connect(self.on_finished)
print("SQL execution is done")
#Then i am calling function to compare two dataframes
class Worker(QtCore.QObject):
finished = QtCore.Signal()
result = QtCore.Signal(object)
#QtCore.Slot(str)
def runsql_MC(self, sqlquery_MC):
print("Thread1 is working")
try:
df1 = pd.read_sql(sql=sqlquery_MC, con=cnxn)
except:
traceback.print_exc()
else:
self.signals.result.emit(df1) # Return the result of the processing
finally:
self.signals.finished.emit() # Done
#QtCore.Slot(str)
def runsql_IRI(self, sqlquery_IRI):
print("Thread2 is working")
try:
df2 = pd.read_sql(sql=sqlquery_IRI, con=cnxn)
except:
traceback.print_exc()
else:
self.signals.result.emit(df2)
finally:
self.signals.finished.emit()
if __name__ == '__main__':
import sys
app = QtWidgets.QApplication(sys.argv)
my_app = mainWindow()
my_app.show()
sys.exit(app.exec_())
self.sqlClicked1.emit(sqlquery1) and self.sqlClicked2.emit(sqlquery2) is calling corresponding threads runsql_MC() and runsql_IRI. Then I need to wait till two threads are completed to start comparison process. Currently its not happening.

Although your code is not an MRE, show your ignorance of various concepts.
The emission of a signal does not imply obtaining the data as a result since it will be sent asynchronously.
In your code even if you invoke 2 queries does not imply that each one runs on different threads since the worker lives in a single thread.
Your runsql_MC and runsql_IRI methods are redundant since they are a template of the same thing.
In addition to other errors such as that there is no object/signal called sqlClicked, you have not declared the object signals, etc.
The idea is to have a worker who lives in a different thread for each query, and create a class that handles the workers waiting for the data and eliminating when they have finished their work.
from functools import partial
import sqlite3
import pandas as pd
from PySide2 import QtCore, QtGui, QtWidgets
class Worker(QtCore.QObject):
finished = QtCore.Signal()
result = QtCore.Signal(object)
#QtCore.Slot(str)
def runsql(self, query):
cnxn = sqlite3.connect("test.db")
print("Thread1 is working")
try:
df1 = pd.read_sql(sql=query, con=cnxn)
except:
traceback.print_exc()
else:
self.result.emit(df1) # Return the result of the processing
finally:
self.finished.emit() # Done
class SqlManager(QtCore.QObject):
results = QtCore.Signal(list)
def __init__(self, parent=None):
super().__init__(parent)
self.workers_and_threads = {}
self.dataframes = []
def execute_queries(self, queries):
for query in queries:
thread = QtCore.QThread(self)
thread.start()
worker = Worker()
worker.result.connect(self.onResults)
worker.moveToThread(thread)
self.workers_and_threads[worker] = thread
# launch task
wrapper = partial(worker.runsql, query)
QtCore.QTimer.singleShot(0, wrapper)
#QtCore.Slot(object)
def onResults(self, result):
worker = self.sender()
thread = self.workers_and_threads[worker]
thread.quit()
thread.wait()
del self.workers_and_threads[worker]
worker.deleteLater()
self.dataframes.append(result)
if not self.workers_and_threads:
self.results.emit(self.dataframes)
self.dataframes = []
class MainWindow(QtWidgets.QMainWindow):
def __init__(self, parent=None):
super().__init__(parent)
self.push_button = QtWidgets.QPushButton("Run Report")
self.push_button.clicked.connect(self.run_report)
self.setCentralWidget(self.push_button)
self.manager = SqlManager(self)
self.manager.results.connect(self.onResults)
#QtCore.Slot()
def run_report(self):
sqlquery1 = "Select * from table1"
sqlquery2 = "Select * from table2"
queries = [sqlquery1, sqlquery2]
self.manager.execute_queries(queries)
self.push_button.setEnabled(False)
#QtCore.Slot(list)
def onResults(self, dataframes):
print(dataframes)
self.push_button.setEnabled(True)
if __name__ == "__main__":
import sys
app = QtWidgets.QApplication(sys.argv)
my_app = MainWindow()
my_app.show()
sys.exit(app.exec_())

I'm not quite sure if it is written in your code but Have you ever uses a join() sentence to wait till those threads end? You should be using it when you awake those two threads inside your def __init__() method from your mainWindow class

QTableView - How to handle a lot of dataChanged signals?

I have to program a table view that should be able to handle tens thousands of cells containing images (image is different for every cell). All in python using PySide2.
I implemented the loading of my images using a thread pool.
The problem is that I have to asynchronusly notify the view that the image has been loaded for a given index so it can reload the display. Using the dataChanged signal works but there are just too many of them to process and the UI does not show up until all the indexes are processed by the thread pool.
I provide a working example below that reproduces the issue (no images, juste text).
For now, I solved the problem by letting the threads sleep a little (just uncomment the time.sleep(1) line in the Work.run method) but it feels more like a dirty hack than a real solution to me.
I thought about the following solutions:
Try to make dataChanged signal behave asynchronusly. I suspect the default connection between dataChanged and whatever slot that update the view to be a AutoConnection. Is there any way to do this?
Gather the modified indexes in a buffer and update the view at regular time intervals. I would like to avoid this solution because finding a good time interval between two evaluation of the buffer is quite a hard task.
Do you have any other idea on how to avoid this blocking behavior?
Thank you for your advice!
import math
from random import choice
import string
import time
from PySide2.QtCore import QModelIndex
from PySide2.QtCore import Qt
from PySide2.QtCore import QObject
from PySide2.QtCore import Signal
from PySide2.QtCore import QThread
from PySide2.QtCore import QThreadPool
from PySide2.QtCore import QMutex
from PySide2.QtCore import QAbstractTableModel
from PySide2.QtWidgets import QTableView
class Notifier(QObject):
finished = Signal(QModelIndex, str)
class Work(QThread):
def __init__(self, *args, **kwargs):
super(Work, self).__init__(*args, **kwargs)
self.work = []
self._stopped = False
self._slept = False
def run(self):
while True:
try:
work = self.work.pop(0)
except IndexError:
work = None
if not work:
if self._slept:
break
self.msleep(500)
self._slept = True
continue
# Uncomment the following line to make the UI responsive
# time.sleep(1)
if work[0]:
c = ''.join(choice(string.ascii_uppercase + string.digits)
for _ in range(6))
work[0].finished.emit(work[1], c)
def reset(self):
self.work = []
self._stopped = True
class WorkPool(object):
def __init__(self):
self.thread_count = QThreadPool().maxThreadCount()
self.thread_pool = []
self.thread_cpt = 0
self.mutex = QMutex()
for c in range(0, self.thread_count):
self.thread_pool.append(Work())
def add_work(self, notifier, index):
new_thread = divmod(self.thread_cpt, self.thread_count)[1]
thread = self.thread_pool[new_thread]
self.thread_cpt += 1
thread.work.append((notifier, index))
if not thread.isRunning():
thread.start()
def terminate(self):
self.mutex.lock()
for t in self.thread_pool:
t.reset()
for t in self.thread_pool:
t.wait()
self.mutex.unlock()
class TableModel(QAbstractTableModel):
def __init__(self, items, *args, **kwargs):
super(TableModel, self).__init__(*args, **kwargs)
self.items = items
self.works = []
self.loader = WorkPool()
def index(self, row, column, parent=QModelIndex()):
pos = row * self.columnCount() + column
try:
return self.createIndex(row, column,self.items[pos])
except IndexError:
return QModelIndex()
def data(self, index, role):
if not index.isValid():
return None
if role == Qt.DisplayRole:
return index.internalPointer()
def columnCount(self, parent=QModelIndex()):
return 10
def rowCount(self, parent=QModelIndex()):
return int(math.ceil(float(len(self.items)) / self.columnCount()))
def refresh_content(self):
# Launch a thread to update the content of each index
for r in range(0, self.rowCount()):
for c in range(0, self.columnCount()):
index = self.index(r, c)
notifier = Notifier()
notifier.finished.connect(self.setData)
self.loader.add_work(notifier, index)
def setData(self, index, value):
if not index.isValid():
return False
self.items[index.row() * self.columnCount() + index.column()] = value
self.dataChanged.emit(index, index)
return True
class TableView(QTableView):
def closeEvent(self, *args, **kwargs):
self.model().loader.terminate()
super(TableView, self).closeEvent(*args, **kwargs)
if __name__ == '__main__':
from PySide2.QtWidgets import QApplication
app = QApplication([])
tv = TableView()
model = TableModel([None] * 99999)
tv.setModel(model)
model.refresh_content()
tv.show()
app.exec_()

get response or signal when all threads are complete

Using GUI i am copying multiple files. for each file i am starting a thread. Is there is a way with which I can get a signal or response which tells that all the threads are done with their work. I need the response because depending on the response when all threads are done, I want to enable/disable few buttons and check boxes on the GUI.
for file_name in _src_files:
if (os.path.exists(_full_file_name)):
t = Thread(target=file_util.copy_file, args=[_full_file_name, _str_destination_dir, 'update=1'])
t.start()

You must create a class that inherits from QObject and have custom signals, for example:
def copy_file(source, destination, message, log):
[...]
log.finished.emit()
class Logger(QObject):
finished = pyqtSignal()
self.l = Logger()
for file_name in _src_files:
if (os.path.exists(_full_file_name)):
t = Thread(target=file_util.copy_file, args=[_full_file_name, _str_destination_dir, 'update=1'], self.l)
t.start()
self.l.finished.connect(some_slot)
Example:
import time
from threading import Thread
import sys
from PyQt4.QtGui import *
from PyQt4.QtCore import *
class Logger(QObject):
finished = pyqtSignal(str)
def myfunc(i, log):
print("sleeping 5 sec from thread %d" % i)
time.sleep(5)
print("finished sleeping from thread %d" % i)
log.finished.emit("message %d" % i)
class Widget(QWidget):
def __init__(self, *args, **kwargs):
QWidget.__init__(self, *args, **kwargs)
self.l = Logger()
self.counter_threads = 0
for i in range(10):
t = Thread(target=myfunc, args=(i, self.l))
t.start()
self.counter_threads += 1
self.l.finished.connect(self.some_slot)
def some_slot(self, message):
print("some_slot: "+ message)
self.counter_threads -= 1
if self.counter_threads == 0:
print("finished all threads")
if __name__ == '__main__':
app = QApplication(sys.argv)
w = Widget()
w.show()
sys.exit(app.exec_())