I am newbie in Threading and I am writing a simple program which can do two tasks simultaneously. One function will record audio for arbitrary duration and the other function will just print some messages. In the record function, the Ctrl+C keyboard interrupt is used to stop the recording. But, if I put this function into a python thread, the Ctrl+C is not working and I am not able to stop the recording. The code is as follows. Any help is kindly appreciated.
import argparse
import tempfile
import queue
import sys
import threading
import soundfile as sf
import sounddevice as sd
def callback(indata, frames, time, status):
"""
This function is called for each audio block from the record function.
"""
if status:
print(status, file=sys.stderr)
q.put(indata.copy())
def record(filename='sample_audio.wav'):
"""
Audio recording.
Args:
filename (str): Name of the audio file
Returns:
Saves the recorded audio as a wav file
"""
q = queue.Queue()
try:
# Make sure the file is open before recording begins
with sf.SoundFile(filename, mode='x', samplerate=48000, channels=2, subtype="PCM_16") as file:
with sd.InputStream(samplerate=48000, channels=2, callback=callback):
print('START')
print('#' * 80)
print('press Ctrl+C to stop the recording')
print('#' * 80)
while True:
file.write(q.get())
except OSError:
print('The file to be recorded already exists.')
sys.exit(1)
except KeyboardInterrupt:
print('The utterance is recorded.')
def sample():
for i in range(10):
print('---------------------------------------------------')
def main():
# Threading
thread_1 = threading.Thread(target=record, daemon=True)
thread_2 = threading.Thread(target=sample, daemon=True)
thread_1.start()
thread_2.start()
thread_1.join()
thread_2.join()
if __name__ == "__main__":
main()
Related
I am writing a code to insert data to mongodb every 5 minutes ON and OFF
The problem here is on keyword interrupt my thread should be stop and exit the code execution
Once first record is inserted to DB my time.sleep(300) will make the script sleep
and on my terminal the following line appears -> Press enter to kill the running thread :
Incase If I change my mind don't want to run
simply when I press enter from keyboard the threading which is running and under sleep should be stop and exit
My goal is to stop the thread on the basis of input from user
My code :
import datetime
import threading
import pymongo
import time
from pymongo import MongoClient
dbUrl = pymongo.MongoClient("mongodb://localhost:1245/")
dbName = dbUrl["student"]
dbCollectionName = dbName["student_course"]
def doremon():
return "Hi Friends"
def insert_to_mongodb():
global kill_the_running_thread
while (not kill_the_running_thread):
note_start_time = datetime.datetime.now()
msg = doremon()
note_end_time = datetime.datetime.now()
dt = {"message": msg, "start_time": note_start_time, "end_time": note_end_time}
rec_id1 = dbCollectionName.insert_one(dt)
time.sleep(300)
def main():
global kill_the_running_thread
kill_the_running_thread = False
my_thread = threading.Thread(target=insert_to_mongodb)
my_thread.start()
input("Press enter to kill the running thread : ")
kill_the_running_thread = True
# Calling main
main()
There's a problem when using globals as sentinels in conjunction with sleep. The issue is that the sleep may have only just started (5 minutes in OP's case) and so it could take nearly 5 minutes for the thread to realise that it should terminate.
A preferred (by me) technique is to use a queue. You can specify a timeout on a queue and, of course, it will respond almost immediately to any data passed to it. Here's an example:
from threading import Thread
from queue import Queue, Empty
def process(queue):
while True:
try:
queue.get(timeout=5)
break
except Empty as e:
pass
print('Doing work')
queue = Queue()
thread = Thread(target=process, args=(queue,))
thread.start()
input('Press enter to terminate the thread: ')
queue.put(None)
thread.join()
The process() function will block on the queue for up to 5 seconds (in this example). If there's nothing on the queue it will do its work. If there is something (we just pass None as the trigger), it will terminate immediately
You can try customizing the class, like this:
import datetime
import threading
import pymongo
import time
from pymongo import MongoClient
dbUrl = pymongo.MongoClient("mongodb://localhost:1245/")
dbName = dbUrl["student"]
dbCollectionName = dbName["student_course"]
class ThreadWithKill(threading.Thread):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._target = kwargs.get('target')
self._kill = threading.Event()
self._sleep_duration = 300 # 5 minutes
def run(self):
while True:
# If no kill signal is set, sleep for the duration of the interval.
# If kill signal comes in while sleeping, immediately wake up and handle
self._target() # execute passed function
is_killed = self._kill.wait(self._sleep_duration)
if is_killed:
break
def kill(self):
self._kill.set()
def doremon():
return "Hi Friends"
def insert_to_mongodb():
note_start_time = datetime.datetime.now()
msg = doremon()
note_end_time = datetime.datetime.now()
dt = {"message": msg, "start_time": note_start_time, "end_time": note_end_time}
rec_id1 = dbCollectionName.insert_one(dt)
def main():
my_thread = ThreadWithKill(target=insert_to_mongodb)
my_thread.start()
input("Press enter to kill the running thread : ")
my_thread.kill()
if __name__ == "__main__":
main()
This way there is no need for the kill_the_running_thread variable.
You'll need to test this yourself, because I don't have mongodb.
In this script I was looking to launch a given program and monitor it as long as the program exists. Thus, I reached the point where I got to use the threading's module Timer method for controlling a loop that writes to a file and prints out to the console a specific stat of the launched process (for this case, mspaint).
The problem arises when I'm hitting CTRL + C in the console or when I close mspaint, with the script capturing any of the 2 events only after the time defined for the interval has completely ran out. These events make the script stop.
For example, if a 20 seconds time is set for the interval, once the script has started, if at second 5 I either hit CTRL + C or close mspaint, the script will stop only after the remaining 15 seconds will have passed.
I would like for the script to stop right away when I either hit CTRL + C or close mspaint (or any other process launched through this script).
The script can be used with the following command, according to the example:
python.exe mon_tool.py -p "C:\Windows\System32\mspaint.exe" -i 20
I'd really appreciate if you could come up with a working example.
I had used python 3.10.4 and psutil 5.9.0 .
This is the code:
# mon_tool.py
import psutil, sys, os, argparse
from subprocess import Popen
from threading import Timer
debug = False
def parse_args(args):
parser = argparse.ArgumentParser()
parser.add_argument("-p", "--path", type=str, required=True)
parser.add_argument("-i", "--interval", type=float, required=True)
return parser.parse_args(args)
def exceptionHandler(exception_type, exception, traceback, debug_hook=sys.excepthook):
'''Print user friendly error messages normally, full traceback if DEBUG on.
Adapted from http://stackoverflow.com/questions/27674602/hide-traceback-unless-a-debug-flag-is-set
'''
if debug:
print('\n*** Error:')
debug_hook(exception_type, exception, traceback)
else:
print("%s: %s" % (exception_type.__name__, exception))
sys.excepthook = exceptionHandler
def validate(data):
try:
if data.interval < 0:
raise ValueError
except ValueError:
raise ValueError(f"Time has a negative value: {data.interval}. Please use a positive value")
def main():
args = parse_args(sys.argv[1:])
validate(args)
# creates the "Process monitor data" folder in the "Documents" folder
# of the current Windows profile
default_path: str = f"{os.path.expanduser('~')}\\Documents\Process monitor data"
if not os.path.exists(default_path):
os.makedirs(default_path)
abs_path: str = f'{default_path}\data_test.txt'
print("data_test.txt can be found in: " + default_path)
# launches the provided process for the path argument, and
# it checks if the process was indeed launched
p: Popen[bytes] = Popen(args.path)
PID = p.pid
isProcess: bool = True
while isProcess:
for proc in psutil.process_iter():
if(proc.pid == PID):
isProcess = False
process_stats = psutil.Process(PID)
# creates the data_test.txt and it erases its content
with open(abs_path, 'w', newline='', encoding='utf-8') as testfile:
testfile.write("")
# loop for writing the handles count to data_test.txt, and
# for printing out the handles count to the console
def process_monitor_loop():
with open(abs_path, 'a', newline='', encoding='utf-8') as testfile:
testfile.write(f"{process_stats.num_handles()}\n")
print(process_stats.num_handles())
Timer(args.interval, process_monitor_loop).start()
process_monitor_loop()
if __name__ == '__main__':
main()
Thank you!
I think you could use python-worker (link) for the alternatives
import time
from datetime import datetime
from worker import worker, enableKeyboardInterrupt
# make sure to execute this before running the worker to enable keyboard interrupt
enableKeyboardInterrupt()
# your codes
...
# block lines with periodic check
def block_next_lines(duration):
t0 = time.time()
while time.time() - t0 <= duration:
time.sleep(0.05) # to reduce resource consumption
def main():
# your codes
...
#worker(keyboard_interrupt=True)
def process_monitor_loop():
while True:
print("hii", datetime.now().isoformat())
block_next_lines(3)
return process_monitor_loop()
if __name__ == '__main__':
main_worker = main()
main_worker.wait()
here your process_monitor_loop will be able to stop even if it's not exactly 20 sec of interval
You can try registering a signal handler for SIGINT, that way whenever the user presses Ctrl+C you can have a custom handler to clean all of your dependencies, like the interval, and exit gracefully.
See this for a simple implementation.
This is the solution for the second part of the problem, which checks if the launched process exists. If it doesn't exist, it stops the script.
This solution comes on top of the solution, for the first part of the problem, provided above by #danangjoyoo, which deals with stopping the script when CTRL + C is used.
Thank you very much once again, #danangjoyoo! :)
This is the code for the second part of the problem:
import time, psutil, sys, os
from datetime import datetime
from worker import worker, enableKeyboardInterrupt, abort_all_thread, ThreadWorkerManager
from threading import Timer
# make sure to execute this before running the worker to enable keyboard interrupt
enableKeyboardInterrupt()
# block lines with periodic check
def block_next_lines(duration):
t0 = time.time()
while time.time() - t0 <= duration:
time.sleep(0.05) # to reduce resource consumption
def main():
# launches mspaint, gets its PID and checks if it was indeed launched
path = f"C:\Windows\System32\mspaint.exe"
p = psutil.Popen(path)
PID = p.pid
isProcess: bool = True
while isProcess:
for proc in psutil.process_iter():
if(proc.pid == PID):
isProcess = False
interval = 5
global counter
counter = 0
#allows for sub_process to run only once
global run_sub_process_once
run_sub_process_once = 1
#worker(keyboard_interrupt=True)
def process_monitor_loop():
while True:
print("hii", datetime.now().isoformat())
def sub_proccess():
'''
Checks every second if the launched process still exists.
If the process doesn't exist anymore, the script will be stopped.
'''
print("Process online:", psutil.pid_exists(PID))
t = Timer(1, sub_proccess)
t.start()
global counter
counter += 1
print(counter)
# Checks if the worker thread is alive.
# If it is not alive, it will kill the thread spawned by sub_process
# hence, stopping the script.
for _, key in enumerate(ThreadWorkerManager.allWorkers):
w = ThreadWorkerManager.allWorkers[key]
if not w.is_alive:
t.cancel()
if not psutil.pid_exists(PID):
abort_all_thread()
t.cancel()
global run_sub_process_once
if run_sub_process_once:
run_sub_process_once = 0
sub_proccess()
block_next_lines(interval)
return process_monitor_loop()
if __name__ == '__main__':
main_worker = main()
main_worker.wait()
Also, I have to note that #danangjoyoo's solution comes as an alternative to signal.pause() for Windows. This only deals with CTRL + C problem part. signal.pause() works only for Unix systems. This is how it was supposed for its usage, for my case, in case it were a Unix system:
import signal, sys
from threading import Timer
def main():
def signal_handler(sig, frame):
print('\nYou pressed Ctrl+C!')
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
print('Press Ctrl+C')
def process_monitor_loop():
try:
print("hi")
except KeyboardInterrupt:
signal.pause()
Timer(10, process_monitor_loop).start()
process_monitor_loop()
if __name__ == '__main__':
main()
The code above is based on this.
So I've got a bluetooth connection from an arduino reading a joystick and sending the axis readout over bluetooth to my raspberry pi (4b running ubuntu 20.10). I've confirmed it's receiving this data too.
Now I try to run this bluetooth communcication in a separate process using the python multiprocessing module. to access the data from the arduino, I give the function a queue from the parent main process to put the data in there. Then in the main function I continuously try to read from this queue and process the data there.
The queue in the parent process always remains empty, however, and as such I can't process the data any further.
How can I get the data from the bluetooth process back to the main process?
main.py
#!/usr/bin/env python3
import time
import logging
import multiprocessing as mp
import bluetoothlib
logging.basicConfig(level=logging.DEBUG)
logging.info("creating queue")
global q
q = mp.Queue()
def main():
try:
logging.info("starting bluetooth process")
p = mp.Process(target=bluetoothlib.serlistener, args=(q,))
p.start()
except:
logging.error("unable to start bluetooth listener")
logging.info("start reading from queue")
while True:
#logging.info(q.qsize())
if not q.empty():
mss = q.get()
logging.info(mss)
#do something with data
elif q.empty():
logging.info("queue empty")
time.sleep(1)
main()
bluetoothlib.py
#!/usr/bin/env python3
import os
import serial
import io
def serlistener(q):
print ("creating connection")
btConn = serial.Serial("/dev/rfcomm0", 57600, timeout=1)
btConn.flushInput()
sio = io.TextIOWrapper(io.BufferedRWPair(btConn, btConn, 1),encoding="utf-8")
print ("connection created, starting listening")
while btConn.is_open:
try:
mss = sio.readline()
q.put(mss)
except:
print("error")
break
At thelizardking34's suggestion, I relooked at the global stuff I'd been messing with and after correcting it, the code as now given in the question works.
Thanks to thelizardking34!
I am trying to test a click based threading application with pytest. The application runs forever and waits for a keyboard event.
main.py
#!/usr/bin/python
import threading
import time
import typing
import logging
import click
def test_func(sample_var:str):
print("Sample is: " + sample_var)
#click.option("--sample", required=True, help="Sample String", default="sample", type=str)
#click.command()
def main(sample: str):
print("Starting App r")
interrupt = False
while not interrupt:
try:
print("Start threading ..")
my_thread = threading.Thread(
target=test_func,
kwargs=dict(sample_var=sample),
daemon=True)
my_thread.start()
my_thread.join(120)
if not interrupt:
print("Resting for 60 seconds")
time.sleep(60)
except(KeyboardInterrupt, SystemExit):
print("Received Keyboard Interrupt or system exisying, cleaning all the threads")
interrupt=True
if __name__ == "__main__":
main(auto_envvar_prefix="MYAPP")
The problem is that while testing I do not know how to send the Keyboard Interrupt Signal
main_test.py
from click.testing import CliRunner
from myapp.main import main
import pytest
import time
import click
def test_raimonitor_failing(cli_runner):
""" Tests the Startup off my app"""
runner = CliRunner()
params = ["--sample", "notsample"]
test = runner.invoke(cli = main, args = params)
expected_msg = "notsample\n"
print(test.stdout)
print(test.output)
assert 0
assert expected_msg in test.stdout
And the tests just hangs, and I don't know how to send the signal to stop it.
How can I test this system properly?
To test a KeyboardInterrupt exception handler in a click function, you can use side_effect on a Mock
from unittest import mock
with mock.patch('test_code.wait_in_loop', side_effect=KeyboardInterrupt):
result = runner.invoke(cli=main, args=params)
To make the testing easier, the time.sleep() call was moved into a separate function, and then that function was mocked.
Test Code
from unittest import mock
def test_raimonitor_failing():
""" Tests the Startup off my app"""
runner = CliRunner()
params = ["--sample", "notsample"]
with mock.patch('test_code.wait_in_loop', side_effect=KeyboardInterrupt):
result = runner.invoke(cli=main, args=params)
expected = '\n'.join(line.strip() for line in """
Starting App
Start threading ..
Sample is: notsample
Resting for 60 seconds
Received Keyboard Interrupt or system exiting, cleaning all the threads
""".split('\n')[1:-1])
assert result.output == expected
Code Under Test
from click.testing import CliRunner
import click
import threading
import time
def a_test_func(sample_var: str):
print("Sample is: " + sample_var)
def wait_in_loop():
time.sleep(60)
#click.option("--sample", required=True, help="Sample String", default="sample", type=str)
#click.command()
def main(sample: str):
print("Starting App")
interrupt = False
while not interrupt:
try:
print("Start threading ..")
my_thread = threading.Thread(
target=a_test_func,
kwargs=dict(sample_var=sample),
daemon=True)
my_thread.start()
my_thread.join(120)
if not interrupt:
print("Resting for 60 seconds")
wait_in_loop()
except(KeyboardInterrupt, SystemExit):
print("Received Keyboard Interrupt or system exiting, cleaning all the threads")
interrupt = True
I'm trying to use zc.lockfile. I see that a lockfile is created in the same directory as my python script, but when I press ctrl+C, the file is NOT removed. I have a callback registered and have even tested given a long time (not sure if zc.lockfile spawns a new thread and needed time to complete).
import os
import sys
import signal
import time
import zc.lockfile
program_lock = None
def onExitCodePressed(signal, frame):
"""Callback run on a premature user exit."""
global program_lock
print '\r\nYou pressed Ctrl+C'
program_lock.close()
time.sleep(5)
sys.exit(0)
def main():
signal.signal(signal.SIGINT, onExitCodePressed)
if os.path.exists('myapp_lock'):
print "\nAnother instance of the program is already running.\n"
sys.exit(0)
else:
program_lock = zc.lockfile.LockFile('myapp_lock')
while True:
continue
if __name__ == '__main__':
main()