Develop Reference

How can i run sphinxtrain script?

I'm using sphinx to build my own model for vocal recognition , i've followed the tutorial step by step and all works fine until that point when i should run the python script of sphinixtrain (whose role is to execute a set of perl files throw terminal in normal case) but for me, the program only opens the files one by one with a chosen editor without executing them !(watching other tutorials videos, the code bellow works normal)
The code of trainer :
#!/usr/bin/python
from __future__ import print_function
import getopt, sys, os
training_basedir = ""
sphinxbinpath = ""
sphinxpath = ""
def find_paths():
global training_basedir
global sphinxbinpath
global sphinxpath
# Find the location of the files, it can be libexec or lib or lib64
currentpath = os.path.dirname(os.path.realpath(__file__))
sphinxbinpath = os.path.realpath(currentpath + "/../libexec/sphinxtrain")
if os.path.exists(currentpath + "/../lib/sphinxtrain/bw"):
sphinxbinpath = os.path.realpath(currentpath + "/../lib/sphinxtrain/bw")
if os.path.exists(currentpath + "/../bin/Release/Win32"):
sphinxbinpath = os.path.realpath(currentpath + "/../bin/Release/Win32")
# Find the location for the libraries
sphinxpath = os.path.realpath(currentpath + "/../lib/sphinxtrain")
if os.path.exists(currentpath + "/../lib64/sphinxtrain/scripts/00.verify"):
sphinxpath = os.path.realpath(currentpath + "/../lib64/sphinxtrain")
if os.path.exists(currentpath + "/../scripts/00.verify"):
sphinxpath = os.path.realpath(currentpath + "/..")
if not (os.path.exists(sphinxbinpath + "/bw") or os.path.exists(sphinxbinpath + "/bw.exe")):
print("Failed to find sphinxtrain binaries. Check your installation")
exit(1)
# Perl script want forward slashes
training_basedir = os.getcwd().replace('\\', '/');
sphinxpath = sphinxpath.replace('\\','/')
sphinxbinpath = sphinxbinpath.replace('\\','/')
print("Sphinxtrain path:", sphinxpath)
print("Sphinxtrain binaries path:", sphinxbinpath)
def setup(task):
if not os.path.exists("etc"):
os.mkdir("etc")
print("Setting up the database " + task)
out_cfg = open("./etc/sphinx_train.cfg", "w")
for line in open(sphinxpath + "/etc/sphinx_train.cfg", "r"):
line = line.replace("___DB_NAME___", task)
line = line.replace("___BASE_DIR___", training_basedir)
line = line.replace("___SPHINXTRAIN_DIR___", sphinxpath)
line = line.replace("___SPHINXTRAIN_BIN_DIR___", sphinxbinpath)
out_cfg.write(line)
out_cfg.close()
out_cfg = open("etc/feat.params", "w")
for line in open(sphinxpath + "/etc/feat.params", "r"):
out_cfg.write(line)
out_cfg.close()
steps = [
"000.comp_feat/slave_feat.pl",
"00.verify/verify_all.pl",
"0000.g2p_train/g2p_train.pl",
"01.lda_train/slave_lda.pl",
"02.mllt_train/slave_mllt.pl",
"05.vector_quantize/slave.VQ.pl",
"10.falign_ci_hmm/slave_convg.pl",
"11.force_align/slave_align.pl",
"12.vtln_align/slave_align.pl",
"20.ci_hmm/slave_convg.pl",
"30.cd_hmm_untied/slave_convg.pl",
"40.buildtrees/slave.treebuilder.pl",
"45.prunetree/slave.state-tying.pl",
"50.cd_hmm_tied/slave_convg.pl",
"60.lattice_generation/slave_genlat.pl",
"61.lattice_pruning/slave_prune.pl",
"62.lattice_conversion/slave_conv.pl",
"65.mmie_train/slave_convg.pl",
"90.deleted_interpolation/deleted_interpolation.pl",
"decode/slave.pl",
]
def run_stages(stages):
for stage in stages.split(","):
for step in steps:
name = step.split("/")[0].split(".")[-1]
if name == stage:
ret = os.system(sphinxpath + "/scripts/" + step)
if ret != 0:
exit(ret)
def run_from(stage):
found = False
for step in steps:
name = step.split("/")[0].split(".")[-1]
if name == stage or found:
found = True
ret = os.system(sphinxpath + "/scripts/" + step)
if ret != 0:
exit(ret)
def run():
print("Running the training")
for step in steps:
ret = os.system(sphinxpath + "/scripts/" + step)
if ret != 0:
exit(ret)
def usage():
print ("")
print ("Sphinxtrain processes the audio files and creates and acoustic model ")
print ("for CMUSphinx toolkit. The data needs to have a certain layout ")
print ("See the tutorial http://cmusphinx.sourceforge.net/wiki/tutorialam ")
print ("for details")
print ("")
print ("Usage: sphinxtrain [options] <command>")
print ("")
print ("Commands:")
print (" -t <task> setup - copy configuration into database")
print (" [-s <stage1,stage2,stage3>] [-f <stage>] run - run the training or just selected
stages")
def main():
try:
opts, args = getopt.getopt(sys.argv[1:], "ht:s:f:", ["help", "task", "stages", "from"])
except getopt.GetoptError as err:
print(str(err))
usage()
sys.exit(-1)
task = None
stages = None
from_stage = None
for o, a in opts:
if o in ("-t", "--task"):
task = a
if o in ("-f", "--from"):
from_stage = a
if o in ("-s", "--stages"):
stages = a
if o in ("-h", "--help"):
usage()
if len(args) == 0:
usage()
sys.exit(-1)
command = args[0]
find_paths()
if command == "setup":
if task == None:
print("No task name defined")
sys.exit(-1)
setup(task)
elif command == "run":
if stages != None:
run_stages(stages)
elif from_stage != None:
run_from(from_stage)
else:
run()
else:
run()
if __name__ == "__main__":
main()

Another way to solve this is to be explicit about calling the Perl interpreter in your os.system call
i.e.
ret = os.system('perl ' + sphinxpath + "/scripts/" + step)

Solved by associating perl files ( with pl extension) to perl.exe

how to print variables after to use subprocess.call?

In my program, I use subprocess.call to run apache drill automatically.
After that, I make some queries and I would like to print the result.
Before to program the code to run apache drill automatically, I was doing it manually and i could print the results but now i cannot do it.
My last try was to write in a file, but the behavior is the same, nothing is written.
My code is bellow.
import subprocess
from pydrill.client import PyDrill
import sys
writer = open('resultado.txt', 'w')
cmdmsg = subprocess.check_output("cd C:\\Users\\Tito\\Downloads\\apache-drill-1.14.0\\bin & sqlline -u \"jdbc:drill:zk=local\"", shell = True)
writer.write("teste de msg: " + str(cmdmsg))
drill = PyDrill(host='localhost', port=8047)
if drill.is_active:
result = drill.query('''SELECT * FROM cp.`employee.json` LIMIT 3''')
result2 = drill.query('''SELECT * FROM dfs.`/Users/Tito/Desktop/banco_gal.csv` LIMIT 5''')
for tuple in result2:
writer.write(tuple)
writer.close

I could solve this problem.
3 things are important for this topic.
a) we should kill the java virtual machine and shell after that apache drill is turned on.
b) the windows buffer is very short, so the result has not been printed.
c) popen method is better than call for this task.
import os
import re
import subprocess
import traceback
from os import path
from pydrill.client import PyDrill
DRILL_HOST = 'localhost'
DRILL_PORT = 8047
JAVA_HOME = ''
JAVA_HOME = JAVA_HOME or ("JAVA_HOME" in os.environ and os.environ["JAVA_HOME"])
JPS_EXE_PATH = path.join(JAVA_HOME, 'bin', 'jps.exe')
KILL_DRILL_JVM = True
def getJvmPID(className):
pid = None
print('Running JPS cmd: %s' % JPS_EXE_PATH)
jpsOutput = subprocess.check_output(JPS_EXE_PATH)
jpsOutput = str(jpsOutput)
se = re.search(r'([0-9]*\s*)' + className, jpsOutput)
if se:
pid = se.group(1)
return pid
def killProcessByPID(pid):
killCmd = ['taskkill', '/f', '/pid', str(pid)]
print('Running taskkill cmd: %s' % killCmd)
killCmdOuput = subprocess.check_output(killCmd, stderr=subprocess.STDOUT)
print(str(killCmdOuput))
def killJvm(className):
pid = getJvmPID(className)
killProcessByPID(pid)
drillBinDir = 'C:/Users/Tito/Downloads/apache-drill-1.14.0/bin'
sqlinePath = path.join(drillBinDir, 'sqlline.bat')
drillCmdList = [sqlinePath , '-u', '"jdbc:drill:zk=local"']
drillCmdStr = " ".join(drillCmdList)
drillConAttempts = 2
while drillConAttempts > 0:
drillConAttempts -= 1
print("Connecting to drill on %s:%d..." % (DRILL_HOST, DRILL_PORT))
try:
drill = PyDrill(host=DRILL_HOST, port=DRILL_PORT)
except:
print("Exception when creating object")
traceback.print_exc()
print("Checking Drill conection...")
try:
if drill.is_active():
print("Connected.")
break
elif drillConAttempts > 0:
print("Could not connect to Drill. Trying to start Drill...")
print("Running cmd '%s > %s'" % (drillCmdStr, os.devnull) )
devNull = open(os.devnull,"w")
cmdProc = subprocess.Popen(drillCmdStr, cwd=drillBinDir, stdout=devNull, stderr=subprocess.STDOUT, shell=True)
print("Started CMD process with PID %d" %(cmdProc.pid))
except:
print("Exception when checking connection")
traceback.print_exc()
if drill.is_active():
result = drill.query('''SELECT * FROM cp.`employee.json` LIMIT 3''')
for resultTuple in result:
print(resultTuple)
if KILL_DRILL_JVM:
print('Killing Drill process...')
killJvm('SqlLine')

Multiprocessing show multiple progress bars

For my program, I have a file that writes random integers to a .CSV file.
from __future__ import absolute_import, division, print_function
from numpy.random import randint as randrange
import os, argparse, time
from tqdm import tqdm
def write_to_csv(filename, *args, newline = True):
write_string = ''
for arg in args:
if type(arg) == list:
for i in arg:
write_string += str(i) + ','
else:
write_string += str(arg) + ','
if newline:
write_string = write_string.rstrip(',') + '\n'
else:
write_string = write_string.rstrip(',')
with open(filename+'.csv', 'a') as file:
file.write(write_string)
def move_dir(dirname, parent = False):
if not parent:
dirname = str(dirname)
exists = os.path.isfile(dirname)
try:
os.mkdir(dirname)
os.chdir(dirname)
except FileExistsError:
os.chdir(dirname)
else:
os.chdir("..")
def calculate_probability(odds, exitmode = False, low_cpu = 0):
try:
file_count = 0
move_dir('Probability')
move_dir(str(odds))
d = {}
writelist = []
percentlist = []
for i in tqdm(range(odds)):
d[str(i)] = 0
writelist.append(f'Times {i}')
percentlist.append(f'Percent {i}')
while True:
if os.path.isfile(str(file_count)+'.csv'):
file_count += 1
else:
break
filename = str(file_count)
write_to_csv(filename, 'Number', 'Value')
rep = 500 * odds
if rep > 10000:
rep = 10000
for i in tqdm(range(rep)):
ran = randrange(odds)
ran = int(ran)
d[str(ran)] += 1
if i == 999:
write_to_csv(filename, i, ran+1, newline = False)
else:
write_to_csv(filename, i, ran+1)
if low_cpu:
time.sleep(0.01*float(low_cpu))
writelist2 = []
percentlist2 = []
for i in tqdm(range(odds)):
val = d[str(i)]
writelist2.append(val)
percentlist2.append(round(((val/rep)*100), 2))
if os.path.isfile('runs.csv'):
write_to_csv('runs', file_count, writelist2, percentlist2)
else:
write_to_csv('runs', 'Run #', writelist, percentlist)
write_to_csv('runs', file_count, writelist2, percentlist2)
if exitmode:
exit()
except(KeyboardInterrupt, SystemExit):
if exitmode:
os.remove(str(file_count)+'.csv')
exit()
else:
try:
os.system('cls')
print('User/program interrupted, lauching shutdown mode...')
os.remove(str(file_count)+'.csv')
print('Finilizaing current trial...')
os.chdir("..")
os.chdir("..")
except FileNotFoundError:
exit()
calculate_probability(odds, exitmode = True)
I also have a repetition system to do this multiple times.
def run_tests(times, odds, low_cpu = 0, shutdown = False):
for i in tqdm(range(times)):
calculate_probability(odds, low_cpu = low_cpu)
os.chdir("..")
os.chdir("..")
if shutdown:
os.system('shutdown /S /F /T 0 /hybrid')
However, if I were to run like 30 trails, it would take forever. So I decided to use the multiprocessing module to speed up the process. Because each run needs to write to the same file at the end, I had to collect the data and write them after the processes ended.
def calculate_probability(odds, low_cpu = 0):
try:
file_count = 0
move_dir('Probability')
move_dir(str(odds))
d = {}
writelist = []
percentlist = []
for i in tqdm(range(odds)):
d[str(i)] = 0
writelist.append(f'Times {i}')
percentlist.append(f'Percent {i}')
while True:
if os.path.isfile(str(file_count)+'.csv'):
file_count += 1
else:
break
filename = str(file_count)
write_to_csv(filename, 'Number', 'Value')
rep = 500 * odds
if rep > 10000:
rep = 10000
for i in range(rep):
ran = randrange(odds)
ran = int(ran)
d[str(ran)] += 1
if i == 999:
write_to_csv(filename, i, ran+1, newline = False)
else:
write_to_csv(filename, i, ran+1)
if low_cpu:
time.sleep(0.01*float(low_cpu))
writelist2 = []
percentlist2 = []
for i in range(odds):
val = d[str(i)]
writelist2.append(val)
percentlist2.append(round(((val/rep)*100), 2))
return (writelist, percentlist, writelist2, percentlist2)
except(KeyboardInterrupt, SystemExit):
try:
os.remove(str(file_count)+'.csv')
finally:
exit()
def worker(odds, returndict, num, low_cpu = 0):
returndict[f'write{num}'] = calculate_probability(odds, low_cpu = low_cpu)
os.chdir("..")
os.chdir("..")
os.system('cls')
def run_tests(times, odds, low_cpu = 0, shutdown = False):
print('Starting...')
manager = Manager()
return_dict = manager.dict()
job_list = []
for i in range(times):
p = Process(target=worker, args=(odds,return_dict,i), kwargs = {'low_cpu' : low_cpu})
job_list.append(p)
p.start()
try:
for proc in job_list:
proc.join()
except KeyboardInterrupt:
print('User quit program...')
time.sleep(5)
for proc in job_list:
proc.join()
exit()
else:
move_dir('Probability')
move_dir(str(odds))
if not os.path.isfile('runs.csv'):
write_to_csv('runs', return_dict.values()[0][0], return_dict.values()[0][1])
for value in return_dict.values():
write_to_csv('runs', value[2], value[3])
print('Done!')
finally:
if shutdown:
os.system('shutdown /S /F /T 0 /hybrid')
However, when I run this new code, there is one progressbar, and each process overwrites the bar, so the bar is flashing with random numbers, making the bar useful. I want to have a stack of bars, one for each process, that each update without interrupting the others. The bars do not need to be ordered; I just need to have an idea of how fast each process is doing their tasks.

STDOUT is just a stream, and all of your processes are attached to the same one, so there's no direct way to tell it to print the output from different processes on different lines.
Probably the simplest way to achieve this would be to have a separate process that is responsible for aggregating the status of all the other processes and reporting the results. You can use a multiprocessing.Queue to pass data from the worker threads to the status thread, then the status thread can print the status to stdout. If you want a stack of progress bars, you'll have to get a little creative with the formatting (essentially update all the progress bars at the same time and print them in the same order so they appear to stack up).

How to use multithreading for LCD output on the raspberry pi

Writing to the 16x2 LCD display on the raspberryp pi can take some time to finish, especially with the module I wrote that automatically scrolls text that exceeds the length of the display.
I need to use multithreading, or something similar, to send the output to the display and continue with the rest of the program. I've tried a couple things with multithreading, but haven't quite got it.
This is the working code without any multithreading. The method I want to be multithreaded is "TextToLCD.ProcessFrameBuffer".
piBell.py
#!/usr/bin/env python3
import time
import rekognition
import TextToLCD
import PiPhoto
import json
import logging
import re
import threading
from queue import Queue
logFormatter = logging.Formatter("%(asctime)s [%(name)-8.8s]/[%(funcName)-12.12s] [%(levelname)-5.5s] %(message)s")
rootLogger = logging.getLogger('piBell')
fileHandler = logging.FileHandler("{0}/{1}.log".format("./", "piBell"), 'a')
fileHandler.setFormatter(logFormatter)
rootLogger.addHandler(fileHandler)
consoleHandler = logging.StreamHandler()
consoleHandler.setFormatter(logFormatter)
rootLogger.addHandler(consoleHandler)
reFace = re.compile('face|head|selfie|portrait|person', re.IGNORECASE)
def main(debugMode='INFO'):
TextToLCD.Clear()
rootLogger.setLevel(debugMode)
imgRotation = 270
imgPath = './'
imgName = 'image.jpg'
TextToLCD.ProcessFrameBuffer(["Scanning:", "................."], debugMode)
PiPhoto.GetPhoto(imgPath + imgName, imgRotation, "INFO")
rootLogger.info("Sending image to rekognition.")
TextToLCD.ProcessFrameBuffer(["Processing","................."], debugMode)
jsonLabels = rekognition.get_labels(imgPath + imgName)
rootLogger.info("Obtained JSON payload from rekognition.")
rootLogger.debug(json.dumps(jsonLabels))
if len(json.dumps(jsonLabels)) > 0:
if IsFace(jsonLabels):
if TestFace(imgPath + imgName):
TextToLCD.ProcessFrameBuffer(['Hello', ' :)'], debugMode)
celeb = IsCelebrity(imgPath + imgName)
if celeb:
TextToLCD.ProcessFrameBuffer(["You look like:", celeb], debugMode)
else:
rootLogger.info("No face detected.")
TextToLCD.ProcessFrameBuffer(['No face detected', ' :('], debugMode)
else:
rootLogger.info("No face detected.")
TextToLCD.ProcessFrameBuffer(['No face detected', ' :('], debugMode)
else:
rootLogger.error("JSON payload from rekognition was empty.")
def IsFace(jsonPayload):
for value in jsonPayload:
rootLogger.info("Label: " + value['Name'] + ", Confidence: " + str(round(value['Confidence'])))
rootLogger.debug(json.dumps(jsonPayload))
if reFace.match(value['Name']) and round(value['Confidence']) > 75:
rootLogger.info("Possible face match.")
return True
return False
def TestFace(img):
jsonFaces = rekognition.get_faces(img)
rootLogger.debug(json.dumps(jsonFaces))
if len(json.dumps(jsonFaces)) > 2:
for item in jsonFaces:
if item['Confidence']:
if item['Confidence'] > 75:
rootLogger.info("Face detected. Confidence: " + str(round(item['Confidence'])))
return True
else:
rootLogger.info("No facial data obtained.")
return False
def IsCelebrity(img):
celebMatchAccuracy = 25
jsonCelbFaces = rekognition.get_celebrities(img)
rootLogger.debug(json.dumps(jsonCelbFaces))
if len(json.dumps(jsonCelbFaces)) > 2:
for item in jsonCelbFaces:
if item['MatchConfidence']:
if item['MatchConfidence'] > celebMatchAccuracy and item['Name']:
rootLogger.info("Celebirity match detected: " + item['Name'] + ", Confidence: " + str(round(item['MatchConfidence'])))
return item['Name']
else:
rootLogger.info("No celebirity match found.")
return False
if __name__ == "__main__":
main('INFO')

First of all, it would be nice to see your LCD functions.
You are using hardware, so hardware is a limited resource.
For this you will need some kind of access control, this could be implemented with a Lock Object or Event Object.
You have two choices when you used the display:
Run the current writing to the end
Interrupt the current writing
import threading
import time
def parallelWithLock(lock:threading.Lock, name:str):
with lock:
for i in range(5):
print(f"{name}: {i}")
time.sleep(0.5)
# doWantYouWant(...)
def parallelWithInterrupt(event:threading.Event,lock:threading.Lock,name:str):
event.set()
i = 0
with lock:
event.clear()
while True:# or writing
if event.isSet():
print(f"{name} Interrupted!")
break
print(f"{name}: {i}")
time.sleep(1)
i += 1
#doWantYouWant(...)
if __name__ == '__main__':
lock = threading.Lock()
t1 = threading.Thread(target=parallelWithLock,args=(lock,"Thread_1"))
t2 = threading.Thread(target=parallelWithLock,args=(lock,"Thread_2"))
t1.start()
t2.start()
t1.join()
t2.join()
event = threading.Event()
lock = threading.Lock()
t3 = threading.Thread(target=parallelWithInterrupt,args=(event,lock,"Thread_3"))
t4 = threading.Thread(target=parallelWithInterrupt,args=(event,lock,"Thread_4"))
t5 = threading.Thread(target=parallelWithInterrupt, args=(event,lock, "Thread_4"))
t3.start()
time.sleep(5)
t4.start()
time.sleep(3)
t5.start()
t3.join()
t4.join()
time.sleep(2)
event.set()
t5.join()

Unable to catch ctrl+c when running mpi4py

I have been using a code to deal with some data in ~9000 files, since every single process takes some time, I managed to use mpi4py to speed it up. But I cannot make the program catch the ctrl+c signal and save the results already calculated. Here's the code:
import os
import sys
from string import atof
import gc
import pandas as pd
import numpy as np
from StructFunc.Structure_Function import SF_true, SF_fit_params
import scipy.io as sio
import mpi4py
from mpi4py import MPI
import time
import signal
# def handler(signal_num, frame):
# combine_list = comm.gather(p, root=0)
# if comm_rank == 0:
# print combine_list
# combine_dict = {}
# for sub_dict in combine_list:
# for cur_key in sub_dict.keys():
# combine_dict[cur_key] = sub_dict[cur_key]
# print combine_dict
# sio.savemat('./all_S82_DRW_params.mat', combine_dict)
# print "results before the interrupt has been saved."
# sys.exit()
# signal.signal(signal.SIGINT, handler)
comm = MPI.COMM_WORLD
comm_rank = comm.Get_rank()
comm_size = comm.Get_size()
bad_value = [-1, -99.99, -1]
lc_path = '/Users/zhanghaowen/Desktop/AGN/BroadBand_RM/QSO_S82'
model="DRW"
if comm_rank == 0:
file_list = os.listdir(lc_path)
# print file_list
sys.stderr.write('%d files were to be processed.' %len(file_list))
file_list = comm.bcast(file_list if comm_rank == 0 else None, root=0)
num_files = len(file_list)
# num_files = 6
local_files_offset = np.linspace(0, num_files, comm_size+1).astype('int')
local_files = file_list[local_files_offset[comm_rank] : local_files_offset[comm_rank + 1]]
sys.stderr.write('%d/%d processor gets %d/%d data' %(comm_rank, comm_size, len(local_files), num_files))
cnt = 0
p = {}
for ind, lc in enumerate(local_files):
# beginning of process
try:
print local_files[ind]
lc_file = open(os.path.join(lc_path, local_files[ind]), 'r')
lc_data = pd.read_csv(lc_file, header=None, names=['MJD', 'mag', 'err'], usecols=[0, 1, 2], sep=' ')
lc_file.close()
#remove the bad values in the data
lc_data['MJD'] = lc_data['MJD'].replace(to_replace=bad_value[0], value=np.nan)
lc_data['MJD'] = lc_data['MJD'].replace(to_replace=np.nan, value=np.nanmean(lc_data['MJD']))
lc_data['mag'] = lc_data['mag'].replace(to_replace=bad_value[1], value=np.nan)
lc_data['mag'] = lc_data['mag'].replace(to_replace=np.nan, value=np.nanmean(lc_data['mag']))
lc_data['err'] = lc_data['err'].replace(to_replace=bad_value[2], value=np.nan)
lc_data['err'] = lc_data['err'].replace(to_replace=np.nan, value=np.nanmean(lc_data['err']))
MJD = np.array(lc_data['MJD'])
mag = np.array(lc_data['mag'])
err = np.array(lc_data['err'])
SF_params = []
resamp_tag = 0
while resamp_tag < 1:
sim_err = np.array([abs(np.random.normal(0, err[i], size=1)) for i in range(len(err))]).reshape((1, len(err)))[0]
try:
p[lc] = SF_fit_params(MJD, mag, sim_err, MCMC_step=100, MCMC_threads=1, model=model)
cnt += 1
sys.stderr.write('processor %d has processed %d/%d files \n' %(comm_rank, cnt, len(local_files)))
print "finished the MCMC for %s \n" %lc
resamp_tag += 1
except:
continue
# end of process
except KeyboardInterrupt:
combine_list = comm.gather(p, root=0)
if comm_rank == 0:
print combine_list
combine_dict = {}
for sub_dict in combine_list:
for cur_key in sub_dict.keys():
combine_dict[cur_key] = sub_dict[cur_key]
print combine_dict
sio.savemat('./all_S82_DRW_params.mat', combine_dict)
print "save the dict."
os._exit()
combine_list = comm.gather(p, root=0)
if comm_rank == 0:
print combine_list
combine_dict = {}
for sub_dict in combine_list:
for cur_key in sub_dict.keys():
combine_dict[cur_key] = sub_dict[cur_key]
print combine_dict
sio.savemat('./all_S82_DRW_params.mat', combine_dict)
I have tried two ways to catch the ctrl+c signal, i.e. the handler function I defined but commented and the except KeyboardInterrupt trick. When I run the code using python all.py and then type ctrl+c, the script would catch the signal but keeps running after saving the results; while when I use mpirun -np 2 all.py and type ctrl+c, the script just stops without saving.
I think the problem under MPI mode may be that only the manager process catched the signal, but why didn't the single process stop in the non-MPI mode after I typed ctrl+c? Do anyone know how to make the worker process catch the signal and do something before terminating?
By the way I'm using openmpi.