Develop Reference

Python Quart Unable to shutdown background task

I am working on a Python app, but I am moving from Flask to Quart. The application needs a background task that runs constantly whilst the application is running.
When I try to stop the process using control-c, the thread doesn't close cleanly and sits in the while loop in the shutdown routine.
while not self._master_thread_class.shutdown_completed:
if not pro:
print('[DEBUG] Thread is not complete')
pro = True
I have followed this Stackoverflow question, but I can't figure out how to cleanly shutdown the background thread so I would love an explanation please as it seems like the Quart Documentation is lacking a bit.
MasterThread class:
import asyncio
class MasterThread:
def __init__(self, shutdown_requested_event):
self._shutdown_completed = False
self._shutdown_requested_event = shutdown_requested_event
self._shutdown_requested = False
def __del__(self):
print('Thread was deleted')
def run(self, loop) -> None:
asyncio.set_event_loop(loop)
loop.run_until_complete(self._async_entrypoint())
#asyncio.coroutine
def _async_entrypoint(self) -> None:
while not self. _shutdown_requested and \
not self._shutdown_requested_event.isSet():
#print('_main_loop()')
pass
if self._shutdown_requested_event.wait(0.1):
self. _shutdown_requested = True
print('[DEBUG] thread has completed....')
self._shutdown_completed = True
def _main_loop(self) -> None:
print('_main_loop()')
Main application module:
import asyncio
import threading
from quart import Quart
from workthr import MasterThread
app = Quart(__name__)
class Service:
def __init__(self):
self._shutdown_thread_event = threading.Event()
self._master_thread = MasterThread(self._shutdown_thread_event)
self._thread = None
def __del__(self):
self.stop()
def start(self):
loop = asyncio.get_event_loop()
self._thread = threading.Thread(target=self._master_thread.run, args=(loop,))
self._thread.start()
return True
def stop(self) -> None:
print('[DEBUG] Stop signal caught...')
self._shutdown_thread_event.set()
while not self._master_thread.shutdown_completed:
print('[DEBUG] Thread is not complete')
print('[DEBUG] Thread has completed')
self._shutdown()
def _shutdown(self):
print('Shutting down...')
service = Service()
service.start()

Quart has startup and shutdown methods that allow something to be started before the server starts serving and stopped when the server finishes serving. If your background task is mostly IO bound I'd recommend just using a coroutine function rather than a thread,
async def background_task():
while True:
...
#app.before_serving
async def startup():
app.background_task = asyncio.ensure_future(background_task())
#app.after_serving
async def shutdown():
app.background_task.cancel() # Or use a variable in the while loop
Or you can do the same with your Service,
#app.before_serving
async def startup():
service.start()
#app.after_serving
async def shutdown():
service.stop()

Pass data to thread from GUI on runtime in python

I'm trying to build a little python application, where a simple webserver runs in the background, and you can the use GUI to send different messages.
I'm using PyQt5 and Python3.6, and I've managed to pass data from the working thread to the GUI, but I don't know how to do that the other way around.
Here's a skeleton of my code:
MainWindow:
class Ui_MainWindow(object):
def __init__(self):
super().__init__()
self.input = True
# 1 - create Worker and Thread inside the Form
self.obj = WebServer.WebServer(self.input) # no parent!
self.thread = QtCore.QThread() # no parent!
# 2 - Connect Worker`s Signals to Form method slots to post data.
self.obj.dataReady.connect(self.onDataReady)
# 3 - Move the Worker object to the Thread object
self.obj.moveToThread(self.thread)
# 4 - Connect Worker Signals to the Thread slots
self.obj.finished.connect(self.thread.quit)
# 5 - Connect Thread started signal to Worker operational slot method
self.thread.started.connect(self.obj.startServer)
# 6 - Start the thread
self.thread.start()
# 7 - Start the form
self.setupUi()
def setupUi(self):
# Set up the GUI
#...
self.MainWindow.show()
def onDataReady(self, data):
# Data received from working thread
# Do stuff...
def on_click_set2(self):
self.input = not self.input
print(self.input)
if __name__ == "__main__":
import sys
app = QtWidgets.QApplication(sys.argv)
ui = Ui_MainWindow()
sys.exit(app.exec_())
WebServer:
class WebServer(QObject):
finished = pyqtSignal()
dataReady = pyqtSignal(dict)
def __init__(self, input):
super().__init__()
self.input = input
#pyqtSlot()
def startServer(self): # A slot takes no params
# self.loop = asyncio.get_event_loop()
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
coro = asyncio.start_server(self.handle_update, '192.168.2.1', 8888, loop=self.loop)
self.server = self.loop.run_until_complete(coro)
# Serve requests until Ctrl+C is pressed
print('Serving on {}'.format(self.server.sockets[0].getsockname()))
try:
self.loop.run_forever()
except KeyboardInterrupt:
pass
self.finished.emit()
async def handle_update(self, reader, writer):
data = await reader.read(100)
message = data.decode()
addr = writer.get_extra_info('peername')
print(f'Received: {message} from {addr}')
if self.input:
print("Input is True")
else:
print("Input is False")
reply = self.input
print(f'Send: {reply}')
writer.write(str(reply).encode())
await writer.drain()
print("Close the client socket")
writer.close()
self.dataReady.emit(reply)
So for example I want to pass input to the thread, and if I do like above (obviously) input always stays the initial value, and won't change in the thread when I hit the button on the GUI.
How can I do it, that the value of the input is updated whenever I hit the button (so passing a value to the thread from GUI during runtime)? I assume is similar to passing from the thread to the GUI, so emitting a signal from GUI and connecting to it on the tread, but I don't know how to find a reference to the GUI from the working thread.
So any advice on how to do that? And of course any other input regarding the code or approach to the application/background server solution is welcomed! Thanks for the help in advance!
UPDATE:
Maybe it wasn't clear what my question was, so here it is:
How can I send a value from the GUI thread to the worker thread while both of them are running parallel? (If the above code makes any sense to you use that as an example, otherwise a general example would be appreciated)

It is not necessary that the WebServer live in another thread, it is only necessary that the eventloop be executed in another thread. In this case, the WebServer exchanges the data with the server thread through a queue. Although the QObjects are not thread-safe, the signals are so, there is no problem in emitting the signal from a thread other than the one that the QObject lives
import asyncio
import threading
import queue
from functools import partial
from PyQt5 import QtCore, QtGui, QtWidgets
class WebServer(QtCore.QObject):
dataReady = QtCore.pyqtSignal(object)
def startServer(self):
self.m_loop = asyncio.new_event_loop()
self.m_queue = queue.Queue()
asyncio.set_event_loop(self.m_loop)
coro = asyncio.start_server(
self.handle_update, "127.0.0.1", 10000, loop=self.m_loop
)
self.server = self.m_loop.run_until_complete(coro)
print("Serving on {}".format(self.server.sockets[0].getsockname()))
threading.Thread(target=self.m_loop.run_forever, daemon=True).start()
#QtCore.pyqtSlot(object)
def setData(self, data):
if hasattr(self, "m_queue"):
self.m_queue.put(data)
def stop(self):
if hasattr(self, "m_loop"):
self.m_loop.stop()
async def handle_update(self, reader, writer):
reply = ""
data = await reader.read(100)
message = data.decode()
addr = writer.get_extra_info("peername")
print(f"Received: {message} from {addr}")
if not self.m_queue.empty():
data = self.m_queue.get(block=False)
reply = data
print(f"Send: {reply}")
writer.write(str(reply).encode())
await writer.drain()
print("Close the client socket")
writer.close()
self.dataReady.emit(reply)
class Widget(QtWidgets.QWidget):
dataChanged = QtCore.pyqtSignal(object)
def __init__(self, parent=None):
super().__init__(parent)
self.m_lineedit = QtWidgets.QLineEdit()
button = QtWidgets.QPushButton("Send", clicked=self.onClicked)
lay = QtWidgets.QVBoxLayout(self)
lay.addWidget(self.m_lineedit)
lay.addWidget(button)
self.m_web = WebServer()
self.m_web.startServer()
self.dataChanged.connect(self.m_web.setData)
#QtCore.pyqtSlot()
def onClicked(self):
text = self.m_lineedit.text()
self.dataChanged.emit(text)
#QtCore.pyqtSlot(object)
def onDataReady(self, data):
print(data)
def closeEvent(self, event):
self.m_web.stop()
super().closeEvent(event)
if __name__ == "__main__":
import sys
app = QtWidgets.QApplication(sys.argv)
w = Widget()
w.show()
sys.exit(app.exec_())

Signal not handled in PySide

I have an PySide application. In this application, the main function runs in a thread (AnalysisThread). In this thread, I log some stuff with the python logger. I then add a custom logger which essentially triggers a signal with a string. This signal may be handled in the main thread (the GUI thread), but obviously the slots is never triggered, but I am sure that the signal self.messageWritten.emit function is called (debugger confirms this). What am I doing wrong ?
class LogStream(QtCore.QObject):
messageWritten = QtCore.Signal(str)
signal_test = QtCore.Signal()
def write(self, msg):
if not self.signalsBlocked():
self.messageWritten.emit(msg)
class QtHandler(logging.Handler):
def __init__(self, stream):
logging.Handler.__init__(self)
self.stream = stream
def emit(self, record):
record = self.format(record)
if record:
self.stream.write('%s\n' % record)
class AnalysisThread(QtCore.QThread):
processing_ended = QtCore.Signal()
processing_failed = QtCore.Signal(Exception, list)
def __init__(self, analysis):
super(AnalysisThread, self).__init__()
self.analysis = analysis
def run(self):
try:
process = Process(target=self.analysis.analyze)
process.start()
process.join()
except Exception as err:
exec_info = sys.exc_info()
self.processing_failed.emit(err, exec_info)
finally:
self.processing_ended.emit()
class ProcessView(QtGui.QMainWindow):
def __init__(self):
super(ProcessView, self).__init__()
# Log Stream
self.stream = LogStream()
self.stream.messageWritten.connect(self.on_log_written)
def go(self):
analysis = MyAnalysis()
# Handler
handler = QtHandler(self.stream)
handler.setFormatter(logging.Formatter('(%(levelname)s-%(name)s) %(message)s'))
analysis.log.addHandler(handler)
self.processing = AnalysisThread(analysis)
self.processing.processing_ended.connect(self.on_processing_ended)
self.processing.processing_failed.connect(self.on_processing_failed)
self.processing.start()
def on_log_written(self, msg):
print('Message: {}'.format(msg)) # never called
EDIT
For clarification, it is a multithread application, but also a multiprocess one...

Since you're working in a multithreaded environment, try specifying a Qt.QueuedConnection when you're hooking up your signal and slots.
For example:
self.stream.messageWritten.connect(self.on_log_written, QtCore.Qt.QueuedConnection)

ANSWER
I finally found the solution. My application as noted uses multiprocessing, and the log occurs in the child process so the parent process is never alert of the signal. The solution is to create a 'link' between the child and the parent process with a multiprocessing.Pipe. An implementation could be
class Streamer(QtCore.QThread):
messageWritten = QtCore.Signal(str)
def __init__(self, pipe):
super(Streamer, self).__init__()
self.pipe = pipe
def run(self):
while True:
try:
msg = self.pipe.recv()
except EOFError:
break
else:
self.messageWritten.emit(msg)
class QtHandler(logging.Handler):
def __init__(self, stream):
"""Instantiate handler
:param stream: multiprocessing.Pipe
"""
logging.Handler.__init__(self)
self.stream = stream
def emit(self, record):
record = self.format(record)
if record:
self.stream.send('%s\n' % record)
class ProcessView(QtGui.QMainWindow):
def __init__(self):
super(ProcessView, self).__init__()
# Log Stream
mother_pipe, child_pipe = Pipe()
self.stream = child_pipe
self.streamer = Streamer(mother_pipe)
self.streamer.daemon = True
self.streamer.start()
self.streamer.messageWritten.connect(self.on_log_written)
def on_log_written(self, msg):
self.ui.textBrowser_log.insertPlainText(msg)
txt_max = self.ui.textBrowser_log.verticalScrollBar().maximum()
self.ui.textBrowser_log.verticalScrollBar().setValue(txt_max)
Explanations:
The main thread (GUI) starts. At initialization, it creates a multiprocesing.Pipe to communicate with a child proess, and a daemon that listens to the end of the pipe. So after the child process is launched, it eventually logs something. The QtHandler intercepts the message and sends it through the pipe. The daemon receives the message at the other end of the pipe and forwards it to the GUI thread via the signal messageWritten. Finally, this signal is processed by a slot that writes it in a QTextBrowser which scrollbar is refreshed to

Stopping a thread in python, why does the following not work?

I have been trying to provide the functionality to stop a thread following this thread . I have the following class:
class WorkerThread(threading.Thread):
"""Thread class with a stop() method. The thread itself has to check
regularly for the stopped() condition."""
def __init__(self, id):
super(WorkerThread, self).__init__()
self._id = id
self._stop = threading.Event()
def stop(self):
self._stop.set()
def stopped(self):
return self._stop.isSet()
def run(self):
while not self.stopped():
print ("Thread with id {0} Running".format(self._id))
time.sleep(1)
When I create 10 threads and try to stop them, none of them stop and keep printing.
threads = [WorkerThread(x) for x in range(10)]
[thread.start() for thread in threads]
time.sleep(2) #wait
[thread.stop() for thread in threads]
[thread.join() for thread in threads]
print "All done" #never printed
Can someone explain why the stop event is ignored? I also tried using the _stop attribute as a flag and setting it to false in __init__ and True in stop() with no luck.

How to detect if cherrypy is shutting down

I've made a server based on cherrypy but I have a repetitive task which takes a long time (more than a minute) to run. This is all fine until I need to shut down the server, then I am waiting forever for the threads to finish.
I was wondering how you'd detect a cherrypy shutdown inside the client thread so that the thread could abort when the server is shutting down.
I'm after something like this:
class RootServer:
#cherrypy.expose
def index(self, **keywords):
for i in range(0,1000):
lengthyprocess(i)
if server_is_shutting_down():
return

You can inspect the state directly:
if cherrypy.engine.state != cherrypy.engine.states.STARTED:
return
Or, you can register a listener on the 'stop' channel:
class RootServer:
def __init__(self):
cherrypy.engine.subscribe('start', self.start)
cherrypy.engine.subscribe('stop', self.stop)
def start(self):
self.running = True
def stop(self):
self.running = False
#cherrypy.expose
def index(self, **keywords):
for i in range(0,1000):
lengthyprocess(i)
if not self.running:
return
The latter is especially helpful if you also want to have the lengthyprocess start (or perform some preparation) when the server starts up, rather than upon a request.