Develop Reference

Consolidating slots in PyQt5

How can I reduce the number of pyqtSlot() functions so that I don't need to have two of each? It seems like there should be a better way of doing this, but I haven't been able to figure it out.
The code takes two files, reads each file on different threads, and prints the outputs to different QPlainTextEdit objects.
import sys
import time
import traceback
import pandas as pd
from PyQt5.QtWidgets import *
from PyQt5.QtCore import *
from compare_files_gui import Ui_MainWindow
class WorkerSignals(QObject):
"""Defines signals from running worker thread."""
finished = pyqtSignal()
error = pyqtSignal(tuple)
result = pyqtSignal(str)
progress = pyqtSignal(str)
bar = pyqtSignal(int)
class Worker(QRunnable):
"""Worker thread."""
def __init__(self, fn, *args, **kwargs):
super(Worker, self).__init__()
self.fn = fn
self.args = args
self.kwargs = kwargs
self.signals = WorkerSignals()
self.kwargs['progress_callback'] = self.signals.progress
self.kwargs['pbar'] = self.signals.bar
#pyqtSlot()
def run(self):
try:
result = self.fn(*self.args, **self.kwargs)
except:
traceback.print_exc()
exctype, value = sys.exc_info()[:2]
self.signals.error.emit((exctype, value, traceback.format_exc()))
else:
self.signals.result.emit(result)
finally:
self.signals.finished.emit()
class MainWindow(QMainWindow):
def __init__(self):
QMainWindow.__init__(self)
self.ui = Ui_MainWindow()
self.ui.setupUi(self)
self.ui.pushButton.clicked.connect(self.compare)
self.ui.file1_progressBar.setValue(0)
self.ui.file2_progressBar.setValue(0)
self.ui.file1_lineEdit.setText('file1.csv')
self.ui.file2_lineEdit.setText('file2.csv')
self.file1 = self.ui.file1_lineEdit.text()
self.file2 = self.ui.file2_lineEdit.text()
self.threadpool = QThreadPool()
##### How can I consolidate the following slots so I don't need to
##### have 2, one for each console object?
#pyqtSlot(str)
def progress_fn1(self, n):
self.ui.console1_plainTextEdit.appendPlainText(n)
#pyqtSlot(str)
def progress_fn2(self, n):
self.ui.console2_plainTextEdit.appendPlainText(n)
#pyqtSlot(str)
def print_output1(self, s):
self.ui.console1_plainTextEdit.appendPlainText(s)
#pyqtSlot(str)
def print_output2(self, s):
self.ui.console2_plainTextEdit.appendPlainText(s)
#pyqtSlot()
def thread_complete1(self):
self.ui.console1_plainTextEdit.appendPlainText('Processing complete!')
#pyqtSlot()
def thread_complete2(self):
self.ui.console2_plainTextEdit.appendPlainText('Processing complete!')
#pyqtSlot(int)
def update_progress1(self, v):
self.ui.file1_progressBar.setValue(v)
#pyqtSlot(int)
def update_progress2(self, v):
self.ui.file2_progressBar.setValue(v)
def compare(self):
# files = [self.ui.file1_lineEdit.text(), self.ui.file2_lineEdit.text()]
files = [self.file1, self.file2]
# Start new thread for each file
for i, file in enumerate(files, 1):
worker = Worker(self.process_file, file)
#### Is there a better way to do this?
if i == 1:
worker.signals.progress.connect(self.progress_fn1)
worker.signals.result.connect(self.print_output1)
worker.signals.finished.connect(self.thread_complete1)
worker.signals.bar.connect(self.update_progress1)
elif i == 2:
worker.signals.progress.connect(self.progress_fn2)
worker.signals.result.connect(self.print_output2)
worker.signals.finished.connect(self.thread_complete2)
worker.signals.bar.connect(self.update_progress2)
else:
pass
# Execute thread
self.threadpool.start(worker)
def process_file(self, file, pbar, progress_callback):
"""Process file and emit signals."""
t0 = time.time()
progress_callback.emit(f'Processing {file}')
df = pd.read_csv(file, header=None, names=['col'])
num = len(df.index)
for i, (index, row) in enumerate(df.iterrows(), 1):
progress_callback.emit(' ' + row['col'])
pbar.emit(int(i*100/num))
time.sleep(0.25)
t1 = time.time()
return f'Time to complete: {round(t1-t0, 3)} s'
if __name__ == '__main__':
app = QApplication(sys.argv)
main = MainWindow()
main.show()
sys.exit(app.exec_())

To fix the issue, I passed on the QPlainTextEdit and QProgressBar objects associated with each file to the thread, and then emitted those from the thread, so that I could determine which QPlainTextEdit to print to. From what I've read online, I don't think passing UI objects to background threads is a best practice, but it works for this simple application and I haven't been able to figure out a better way to do it.
import sys
import time
import traceback
import pandas as pd
from PyQt5.QtWidgets import *
from PyQt5.QtCore import *
from compare_files_gui import Ui_MainWindow
class WorkerSignals(QObject):
"""Defines signals from running worker thread."""
finished = pyqtSignal(object)
error = pyqtSignal(tuple)
result = pyqtSignal(object, str)
progress = pyqtSignal(object, str)
bar = pyqtSignal(object, int)
class Worker(QRunnable):
"""Worker thread."""
def __init__(self, fn, *args, **kwargs):
super(Worker, self).__init__()
# Store constructor arguments
self.fn = fn
self.args = args
self.kwargs = kwargs
self.signals = WorkerSignals()
# Add callbacks to kwargs
self.kwargs['progress_callback'] = self.signals.progress
self.kwargs['pbar_callback'] = self.signals.bar
#pyqtSlot()
def run(self):
try:
console, result = self.fn(*self.args, **self.kwargs)
except:
traceback.print_exc()
exctype, value = sys.exc_info()[:2]
self.signals.error.emit((exctype, value, traceback.format_exc()))
else:
self.signals.result.emit(console, result)
finally:
self.signals.finished.emit(console)
class MainWindow(QMainWindow):
def __init__(self):
QMainWindow.__init__(self)
self.ui = Ui_MainWindow()
self.ui.setupUi(self)
# Initialize progress bars
self.ui.file1_progressBar.setValue(0)
self.ui.file2_progressBar.setValue(0)
# Connect widgets to slots
self.ui.pushButton.clicked.connect(self.compare)
# Create instance of QThreadPool
self.threadpool = QThreadPool()
#pyqtSlot(object, str)
def progress_fn(self, console, n):
"""Print progress string to specific QPlainTextEdit object."""
console.appendPlainText(n)
#pyqtSlot(object, str)
def print_output(self, console, s):
"""Print result string to specific QPlainTextEdit object."""
console.appendPlainText(s)
#pyqtSlot(object)
def thread_complete(self, console):
"""Print completion text to specific QPlainTextEdit object."""
console.appendPlainText('Processing complete!')
#pyqtSlot(object, int)
def update_progress(self, pbar, v):
"""Set value of QProgressBar object."""
pbar.setValue(v)
def compare(self):
"""Send each file and associated UI objects to thread."""
# Store files in list
# files = [self.ui.file1_lineEdit.text(), self.ui.file2_lineEdit.text()]
files = [self.file1, self.file2]
# Store QPlainTextEdit and QProgressBar objects in list
consoles = [self.ui.console1_plainTextEdit, self.ui.console2_plainTextEdit]
pbars = [self.ui.file1_progressBar, self.ui.file2_progressBar]
# Start new thread for each file
for i, (file, console, pbar) in enumerate(zip(files, consoles, pbars), 1):
worker = Worker(self.process_file, file, console, pbar)
# Connect thread signals to slots in UI thread
worker.signals.progress.connect(self.progress_fn)
worker.signals.result.connect(self.print_output)
worker.signals.finished.connect(self.thread_complete)
worker.signals.bar.connect(self.update_progress)
# Execute thread
self.threadpool.start(worker)
def process_file(self, file, console, pbar, pbar_callback, progress_callback):
"""Process file and emit signals."""
t0 = time.time()
progress_callback.emit(console, f'Processing {file}')
# Read file into dataframe
df = pd.read_csv(file, header=None, names=['col'])
# Iterate over each row and emit value of column and progress
num = len(df.index)
for i, (index, row) in enumerate(df.iterrows(), 1):
progress_callback.emit(console, ' ' + row['col'])
pbar_callback.emit(pbar, int(i*100/num))
# Slow down response
time.sleep(0.25)
t1 = time.time()
# Return QPlainTextEdit object and string
return console, f'Time to complete: {round(t1-t0, 3)} s'
if __name__ == '__main__':
app = QApplication(sys.argv)
main = MainWindow()
main.show()
sys.exit(app.exec_())

PyQt5 run same thread dynamically

I am useing python 3.7 and pyqt5
What I want to do is run same QObject with different thread many times.
here is my main
from PyQt5.QtCore import *
from PyQt5.QtWidgets import *
import threading
import sys
from Worker_Starting import Worker_Starting
class Main(QMainWindow):
def __init__(self) :
super().__init__()
self.setupUI()
self.initSignal()
def setupUI(self):
self.resize(400, 400)
self.pushButton = QPushButton("Start", self)
self.pushButton_2 = QPushButton("Stop", self)
self.pushButton.move(0,0)
self.pushButton_2.move(120,0)
def initSignal(self) :
self.pushButton.clicked.connect(self.Start)
self.pushButton_2.clicked.connect(self.Stop)
#pyqtSlot()
def Start(self) :
for i in range(3) :
print('main', threading.get_ident())
input = ["userID", "userNAME"]
self.my_thread = QThread()
self.my_worker = Worker_Starting(passing_list=input)
self.my_worker.moveToThread(self.my_thread)
self.my_thread.started.connect(self.my_worker.my_fn)
self.my_thread.start()
time.sleep(3)
#pyqtSlot()
def Stop(self) :
pass
if __name__ == "__main__":
app = QApplication(sys.argv)
window = Main()
window.show()
app.exec_()
and my Qobject
from PyQt5 import QtCore
from PyQt5.QtCore import *
import time
import threading
class Worker_Starting(QObject):
def __init__(self, passing_list, parent=None):
QtCore.QThread.__init__(self, parent=parent)
self.userid = passing_list[0]
self.username = passing_list[1]
def my_fn(self):
print(self.username, threading.get_ident())
for i in range(10) :
Now = time.strftime("%Y/%m/%d %H:%M:%S")
print(str(i) + " : " + self.username + " running" + " : " + Now)
time.sleep(0.5)
I put for i in range(3): under def start(self):
because in my actual main script other thread emit input to def start(self): several times.
if I emit only one time, def start(self): works fine.
but when I emit several times, it chrashs.
since I have to emit several times and I need my_thread to work simultaneously and continuously
is there anyway to set same Qobject to different thread?
and how to know or set thread id.
and how to stop thread by thread id.
*I have tried QRunnable, I now it works multi thread. but I have to emit signal back continouously to my main. as far as I know QRunnable are not suitable for custom signal.

The problem is that you're constantly creating new threads by overwriting the existing one, which results in the previous being garbage collected before it's completed, hence the crash.
If you want to execute the same "worker" function concurrently, you cannot use a single object for more threads, as you should use moveToThread each time and this is not possible.
The solution is to use QRunnable instead, implement the function within its run() method, and call QThreadPool to run it.
class Worker_Starting(QRunnable):
def __init__(self, passing_list):
QRunnable.__init__(self)
self.setAutoDelete(False)
self.userid = passing_list[0]
self.username = passing_list[1]
def run(self):
print(self.username, threading.get_ident())
for i in range(10) :
Now = time.strftime("%Y/%m/%d %H:%M:%S")
print(str(i) + " : " + self.username + " running" + " : " + Now)
time.sleep(0.5)
class Main(QMainWindow):
def __init__(self) :
super().__init__()
self.setupUI()
self.initSignal()
self.worker = None
# ...
def Start(self) :
if not self.worker:
input = ["userID", "userNAME"]
self.worker = Worker_Starting(passing_list=input)
for i in range(3):
print('main', threading.get_ident())
QThreadPool.globalInstance().start(self.worker)
Remember that you should not use blocking functions within the main Qt thread (that's why I removed time.sleep(3) from Start()). If you want to start the function more than once at regular intervals, use a QTimer:
interval = 3000 # interval is in milliseconds
for i in range(3):
print('main', threading.get_ident())
QTimer.singleShot(interval * i, lambda:
QThreadPool.globalInstance().start(self.worker))
Note that the pyqtSlot() decorator is usually needed only for special cases and advanced implementation, so you should remove it. Also note that only classes and constants should have capitalized names, while functions, variables and attributes should be lower cased (prefer start and stop instead of Start and Stop).

Python threading.join() hangs

My problem is as follows:
I have a class that inherits from threading.Thread that I want to be able to stop gracefully. This class also has a Queue it get's its work from.
Since there are quite some classes in my project that should have this behaviour, I've created some superclasses to reduce duplicate code like this:
Thread related behaviour:
class StoppableThread(Thread):
def __init__(self):
Thread.__init__(self)
self._stop = Event()
def stop(self):
self._stop.set()
def stopped(self):
return self._stop.isSet()
Queue related behaviour:
class Queueable():
def __init__(self):
self._queue = Queue()
def append_to_job_queue(self, job):
self._queue.put(job)
Combining the two above and adding queue.join() to the stop() call
class StoppableQueueThread(StoppableThread, Queueable):
def __init__(self):
StoppableThread.__init__(self)
Queueable.__init__(self)
def stop(self):
super(StoppableQueueThread, self).stop()
self._queue.join()
A base class for a datasource:
class DataSource(StoppableThread, ABC):
def __init__(self, data_parser):
StoppableThread.__init__(self)
self.setName("DataSource")
ABC.__init__(self)
self._data_parser = data_parser
def run(self):
while not self.stopped():
record = self._fetch_data()
self._data_parser.append_to_job_queue(record)
#abstractmethod
def _fetch_data(self):
"""implement logic here for obtaining a data piece
should return the fetched data"""
An implementation for a datasource:
class CSVDataSource(DataSource):
def __init__(self, data_parser, file_path):
DataSource.__init__(self, data_parser)
self.file_path = file_path
self.csv_data = Queue()
print('loading csv')
self.load_csv()
print('done loading csv')
def load_csv(self):
"""Loops through csv and adds data to a queue"""
with open(self.file_path, 'r') as f:
self.reader = reader(f)
next(self.reader, None) # skip header
for row in self.reader:
self.csv_data.put(row)
def _fetch_data(self):
"""Returns next item of the queue"""
item = self.csv_data.get()
self.csv_data.task_done()
print(self.csv_data.qsize())
return item
Suppose there is a CSVDataSource instance called ds, if I want to stop the thread I call:
ds.stop()
ds.join()
The ds.join() call however, never returns. I'm not sure why this is, because the run() method does check if the stop event is set.
Any Ideas?
Update
A little more clarity as requested: the applications is build up out of several threads. The RealStrategy thread (below) is the owner of all the other threads and is responsible for starting and terminating them. I haven't set the daemon flag for any of the threads, so they should be non-daemonic by default.
The main thread looks like this:
if __name__ == '__main__':
def exit_handler(signal, frame):
rs.stop_engine()
rs.join()
sys.exit(0)
signal.signal(signal.SIGINT, exit_handler)
rs = RealStrategy()
rs.run_engine()
And here are the rs.run_engine() and rs.stop_engine() methods that are called in main:
class RealStrategy(Thread):
.....
.....
def run_engine(self):
self.on_start()
self._order_handler.start()
self._data_parser.start()
self._data_source.start()
self.start()
def stop_engine(self):
self._data_source.stop()
self._data_parser.stop()
self._order_handler.stop()
self._data_source.join()
self._data_parser.join()
self._order_handler.join()
self.stop()

If you want to use queue.Queue.join, then you must also use queue.Queue.task_done. You can read the linked documentation or see the following copied from information available online:
Queue.task_done()
Indicate that a formerly enqueued task is complete.
Used by queue consumer threads. For each get() used to fetch a task, a
subsequent call to task_done() tells the queue that the processing on
the task is complete.
If a join() is currently blocking, it will resume when all items have
been processed (meaning that a task_done() call was received for every
item that had been put() into the queue).
Raises a ValueError if called more times than there were items placed
in the queue.
Queue.join()
Blocks until all items in the queue have been gotten and processed.
The count of unfinished tasks goes up whenever an item is added to the
queue. The count goes down whenever a consumer thread calls
task_done() to indicate that the item was retrieved and all work on it
is complete. When the count of unfinished tasks drops to zero, join()
unblocks.
To test your problem, an example implementation was created to find out what was going on. It is slightly different from how your program works but demonstrates a method to solving your problem:
#! /usr/bin/env python3
import abc
import csv
import pathlib
import queue
import sys
import threading
import time
def main():
source_path = pathlib.Path(r'C:\path\to\file.csv')
data_source = CSVDataSource(source_path)
data_source.start()
processor = StoppableThread(target=consumer, args=[data_source])
processor.start()
time.sleep(0.1)
data_source.stop()
def consumer(data_source):
while data_source.empty:
time.sleep(0.001)
while not data_source.empty:
task = data_source.get_from_queue(True, 0.1)
print(*task.data, sep=', ', flush=True)
task.done()
class StopThread(StopIteration):
pass
threading.SystemExit = SystemExit, StopThread
class StoppableThread(threading.Thread):
def _bootstrap(self, stop=False):
# noinspection PyProtectedMember
if threading._trace_hook:
raise RuntimeError('cannot run thread with tracing')
def terminate():
nonlocal stop
stop = True
self.__terminate = terminate
# noinspection PyUnusedLocal
def trace(frame, event, arg):
if stop:
raise StopThread
sys.settrace(trace)
super()._bootstrap()
def terminate(self):
try:
self.__terminate()
except AttributeError:
raise RuntimeError('cannot terminate thread '
'before it is started') from None
class Queryable:
def __init__(self, maxsize=1 << 10):
self.__queue = queue.Queue(maxsize)
def add_to_queue(self, item):
self.__queue.put(item)
def get_from_queue(self, block=True, timeout=None):
return self.__queue.get(block, timeout)
#property
def empty(self):
return self.__queue.empty()
#property
def full(self):
return self.__queue.full()
def task_done(self):
self.__queue.task_done()
def join_queue(self):
self.__queue.join()
class StoppableQueryThread(StoppableThread, Queryable):
def __init__(self, target=None, name=None, args=(), kwargs=None,
*, daemon=None, maxsize=1 << 10):
super().__init__(None, target, name, args, kwargs, daemon=daemon)
Queryable.__init__(self, maxsize)
def stop(self):
self.terminate()
self.join_queue()
class DataSource(StoppableQueryThread, abc.ABC):
#abc.abstractmethod
def __init__(self, maxsize=1 << 10):
super().__init__(None, 'DataSource', maxsize=maxsize)
def run(self):
while True:
record = self._fetch_data()
self.add_to_queue(record)
#abc.abstractmethod
def _fetch_data(self):
pass
class CSVDataSource(DataSource):
def __init__(self, source_path):
super().__init__()
self.__data_parser = self.__build_data_parser(source_path)
#staticmethod
def __build_data_parser(source_path):
with source_path.open(newline='') as source:
parser = csv.reader(source)
next(parser, None)
yield from parser
def _fetch_data(self):
try:
return Task(next(self.__data_parser), self.task_done)
except StopIteration:
raise StopThread from None
class Task:
def __init__(self, data, callback):
self.__data = data
self.__callback = callback
#property
def data(self):
return self.__data
def done(self):
self.__callback()
if __name__ == '__main__':
main()

Misbehaving QSignals in QThreadPool, QRunnable using QThead on top of it in python

Im stuck in one situation where my first few signals are getting passed and later on no signals are getting emitted.
I'll elaborate it in detail:
I've a job which required heavy processing and ram and can take upto 2-4 hours to complete it without threading in linear way, so I decided to use QTheadPool, so I created (while testing) 355 QRunners which gets started by QThreadPool. And few of them(QRunners) are depends on another QRunners to finish. Its all working fine, I'm able to derive dependencies by emitting signals and catching them. Its all working absolutely perfect when I run this without GUI
For example (below codes are not tested, i just typed here):
from PyQt4 import QtCore
class StreamPool(QtCore.QObject):
def __init__(self, inputs, q_app):
super(....)
self.inputs = inputs
self.q_app = q_app
self.pool = QtCore.QThreadPool()
def start(self):
for each_input in self.inputs:
runner = StreamRunner(each_input, self.q_app)
runner.signal.operation_started.connect(self.mark_as_start)
runner.signal.operation_finished.connect(self.mark_as_start)
self.pool.start(runner)
def mark_as_start(self, name):
print 'operation started..', name
# Some operation...
def mark_as_finish(self, name):
print 'operation finished..', name
# Some operation...
class StreamRunner(QtCore.QRunnable):
def __init__(self, input, q_app):
super(..)
self.input = input
self.q_app = q_app
self.signal = WorkSignals()
def run(self):
self.signal.operation_started.emit(input)
self.q_ap
# Doing some operations
self.signal.operation_finished.emit(input)
self.q_app.processEvents()
class WorkSignals(QtCore.QObject):
operation_started = QtCore.pyqtSignal(str)
operation_finished= QtCore.pyqtSignal(str)
if __name__ == '__main__':
app = QtGui.QApplication([])
st = StreamPool(['a', 'b', 'c'], app)
st.start()
app.exec_()
It works brilliant in above case.
And I want to show the statuses in ui as there can be hundreds of task can be executed, so I wrote a simple ui, and running StreamPool() from another QThread lets say named - StreamWorkThread() which is getting spawned by ui, and StreamWorkThread catching StreamPool.signal.* and sending them back to ui, but in this case StreamPool can only emit few of them, the starting 4 or 5, though the tasks are still getting executed but the dependent task are not getting initialized due to this behavior and no status updates are getting displayed in ui.
I cant share the code with you guys, as its from my work place, I can write similar approach here
class StreamWorkThread (QtCore.QThread):
def __init__(self, inputs, q_app):
super(..)
self.signal = WorkSignals()
self.stream_pool = StreamPool(inputs, q_app)self.stream_pool.signal.operation_started.connect(self.signal.operation_started.emit)
self.stream_pool.signal.operation_finished.connect(self.signal.operation_finished.emit)
def run(self):
self.stream_pool.start()
def print_start(name):
print 'Started --', name
def print_finished(name):
print 'Finished --', name
if __name__ == '__main__':
app = QtGui.QApplication([])
th = StreamWorkThread (['a', 'b', 'c'], app)
th.signal.operation_started.connect(print_start)
th.signal.operation_finshed.connect(print_finished)
th.start()
app.exec_()
Consolidated code:
from PyQt4 import QtCore
class StreamPool(QtCore.QObject):
def __inti__(self, inputs, q_app):
super(StreamPool, self).__init()
self.inputs = inputs
self.q_app = q_app
self.pool = QtCore.QThreadPool()
def start(self):
for each_input in self.inputs:
runner = StreamRunner(each_input, self.q_app)
runner.signal.operation_started.connect(self.mark_as_start)
runner.signal.operation_finished.connect(self.mark_as_start)
self.pool.start(runner)
def mark_as_start(self, name):
print 'operation started..', name
# Some operation...
def mark_as_finish(self, name):
print 'operation finished..', name
# Some operation...
class StreamRunner(QtCore.QRunnable):
def __init__(self, input, q_app):
super(StreamRunner, self).__init()
self.input = input
self.q_app = q_app
self.signal = WorkSignals()
def run(self):
self.signal.operation_started.emit(input)
self.q_ap
# Doing some operations
self.signal.operation_finished.emit(input)
self.q_app.processEvents()
class WorkSignals(QtCore.QObject):
operation_started = QtCore.pyqtSignal(str)
operation_finished= QtCore.pyqtSignal(str)
class StreamWorkThread (QtCore.QThread):
def __init__(self, inputs, q_app):
super(StreamWorkThread, self).__init()
self.signal = WorkSignals()
self.stream_pool = StreamPool(inputs,q_app)
self.stream_pool.signal.operation_started.connect(self.signal.operation_started.emit)
self.stream_pool.signal.operation_finished.connect(self.signal.operation_finished.emit)
def run(self):
self.stream_pool.start()
def print_start(name):
print 'Started --', name
def print_finished(name):
print 'Finished --', name
if __name__ == '__main__':
app = QtGui.QApplication([])
th = StreamWorkThread (['a', 'b', 'c'], app)
th.signal.operation_started.connect(print_start)
th.signal.operation_finshed.connect(print_finished)
th.start()
app.exec_()
Please guys help me, im not getting what exactly the problem here..! :(

Okay, I got the solution for this behavior.
The root cause of blocking the signal is inheriting QObject in StrealPool, when I replace QObject with QThread it worked seamlessly.
Here are the changes I made, only tow places
class StreamPool(**QtCore.QThread**):
def __inti__(self, inputs, q_app):
super(StreamPool, self).__init()
self.inputs = inputs
self.q_app = q_app
self.pool = QtCore.QThreadPool()
def **run**(self):
for each_input in self.inputs:
runner = StreamRunner(each_input, self.q_app)
runner.signal.operation_started.connect(self.mark_as_start)
runner.signal.operation_finished.connect(self.mark_as_start)
self.pool.start(runner)
and that set, it worked ! :D

very difficult without the source code, but the problem is probably when "app.exec_()" is run, the mainloop of the gui is started and interferes with your Streamx classes

How to return value from function running by QThread and Queue

Please explain how do we send/receive data from Thread managed by Queue....
First I subclass 'QThread' defining its run() method which is started when QThread's.start() is called:
class SimpleThread(QtCore.QThread):
def __init__(self, queue, parent=None):
QtCore.QThread.__init__(self, parent)
self.queue=queue
def run(self):
while True:
arg=self.queue.get()
self.fun(arg)
self.queue.task_done()
def fun(self, arg):
for i in range (3):
print 'fun: %s'%i
self.sleep(1)
return arg+1
Then I declare two Thread instances (so only two CPU cores are taken) sending self.queue instance as an argument.
self.queue=queue.Queue()
for i in range(2):
thread=SimpleThread(self.queue)
thread.start()
Now if I understand it correctly thread.start() is not starting anything. The real "start" happens only when I call queue.put():
for arg in [1,2,3]: self.queue.put(arg)
This last line is what makes a "real" call. Aside from creating and starting Queue item put() allows to save any arbitrary value to each Queue item. .put() does several things at once: it creates, it starts, it moves the processing through the Queue and it allows to place a variable "inside" of the queue item (which later can be retrieved from inside of the function-processor: using Queue item's '.get()` method).
But how do I return the value from fun() function. A "regular" fun()'s return resultValue doesn't work. And I can't use self.queue.put() method since this method aside from storing a data "creates" a new queue item...
EDITED LATER:
Here is slightly tweaked code (copy/pasted from another post) showing an approach on how to return a value from completed Thread. I am not sure if the the approach used here would work with QThread... please correct me if I am wrong:
import os, sys
import threading
import Queue
def callMe(incomingFun, daemon=False):
def execute(_queue, *args, **kwargs):
result=incomingFun(*args, **kwargs)
_queue.put(result)
def wrap(*args, **kwargs):
_queue=Queue.Queue()
_thread=threading.Thread(target=execute, args=(_queue,)+args, kwargs=kwargs)
_thread.daemon=daemon
_thread.start()
_thread.result_queue=_queue
return _thread
return wrap
#callMe
def localFunc(x):
import time
x = x + 5
time.sleep(5)
return x
thread=localFunc(10)
# this blocks, waiting for the result
result = thread.result_queue.get()
print result

In normal circumstances you'd use a result queue to send results back, and then have some other thread running that waits for the results:
class SimpleThread(QtCore.QThread):
def __init__(self, queue, result_queue, parent=None):
QtCore.QThread.__init__(self, parent)
self.queue=queue
self.result_queue = result_queue
def run(self):
while True:
arg=self.queue.get()
self.fun(arg)
self.queue.task_done()
def fun(self, arg):
for i in range (3):
print 'fun: %s'%i
self.sleep(1)
self.result_queue.put(arg+1)
def handle_results(result_queue):
while True:
result = result_queue.get()
print("Got result {}".format(result))
Main thread:
self.queue=queue.Queue()
self.result_queue = queue.Queue()
result_handler = threading.Thread(target=handle_results, self.result_queue)
for i in range(2):
thread=SimpleThread(self.queue, self.result_queue)
thread.start()
Doing it this way will keep you from blocking the GUI's event loop while you wait for the results. Here's what the equivalent would look like with multiprocessing.pool.ThreadPool:
from multiprocessing.pool import ThreadPool
import time
def fun(arg):
for i in range (3):
print 'fun: %s'%i
time.sleep(1)
return arg+1
def handle_result(result):
print("got result {}".format(result))
pool = ThreadPool(2)
pool.map_async(fun, [1,2,3], callback=handle_result)
Which is a lot simpler. It internally creates a result handling thread, which will automatically call handle_result for you when fun completes.
That said, you're using QThread, and you want the results to update GUI widgets, so you really want your results to be sent back to the main thread, not to a result handling thread. In that case, it makes sense to use Qt's signaling system, so that you can safely update the GUI when you receive the result:
from PyQt4 import QtCore, QtGui
import sys
import Queue as queue
class ResultObj(QtCore.QObject):
def __init__(self, val):
self.val = val
class SimpleThread(QtCore.QThread):
finished = QtCore.pyqtSignal(object)
def __init__(self, queue, callback, parent=None):
QtCore.QThread.__init__(self, parent)
self.queue = queue
self.finished.connect(callback)
def run(self):
while True:
arg = self.queue.get()
if arg is None: # None means exit
print("Shutting down")
return
self.fun(arg)
def fun(self, arg):
for i in range(3):
print 'fun: %s' % i
self.sleep(1)
self.finished.emit(ResultObj(arg+1))
class AppWindow(QtGui.QMainWindow):
def __init__(self):
super(AppWindow, self).__init__()
mainWidget = QtGui.QWidget()
self.setCentralWidget(mainWidget)
mainLayout = QtGui.QVBoxLayout()
mainWidget.setLayout(mainLayout)
button = QtGui.QPushButton('Process')
button.clicked.connect(self.process)
mainLayout.addWidget(button)
def handle_result(self, result):
val = result.val
print("got val {}".format(val))
# You can update the UI from here.
def process(self):
MAX_CORES=2
self.queue = queue.Queue()
self.threads = []
for i in range(MAX_CORES):
thread = SimpleThread(self.queue, self.handle_result)
self.threads.append(thread)
thread.start()
for arg in [1,2,3]:
self.queue.put(arg)
for _ in range(MAX_CORES): # Tell the workers to shut down
self.queue.put(None)
app = QtGui.QApplication([])
window = AppWindow()
window.show()
sys.exit(app.exec_())
Output when the button is pushed:
fun: 0
fun: 0
fun: 1
fun: 1
fun: 2
fun: 2
fun: 0
got val 2
got val 3
Shutting down
fun: 1
fun: 2
Shutting down
got val 4