Develop Reference

subprocess or multithreading or threading pool

I'm going to run the command line utility multiple times in parallel using Python.
I know that multithreading is better to use for I/O operations, multiprocessing - for CPU oriented operations.
But what should I use for parallel subprocess.run?
I also know that I can create a pool from the subprocess module, but how is it different from pools from the multiprocessing and threading modules? And why shouldn't I just put subprocess.run function into multiprocessing or threading pools?
Or maybe there are some criteria when it is better to put a utility run cmd into a pool of threads or processes?
(In my case, I'm going to run the "ffmpeg" utility)

In a situation like this, I tend to run subprocesses from a ThreadPoolExecutor, basically because it's easy.
Example (from here):
from datetime import datetime
from functools import partial
import argparse
import concurrent.futures as cf
import logging
import os
import subprocess as sp
import sys
__version__ = "2021.09.19"
def main():
"""
Entry point for dicom2jpg.
"""
args = setup()
if not args.fn:
logging.error("no files to process")
sys.exit(1)
if args.quality != 80:
logging.info(f"quality set to {args.quality}")
if args.level:
logging.info("applying level correction.")
convert_partial = partial(convert, quality=args.quality, level=args.level)
starttime = str(datetime.now())[:-7]
logging.info(f"started at {starttime}.")
with cf.ThreadPoolExecutor(max_workers=os.cpu_count()) as tp:
for infn, outfn, rv in tp.map(convert_partial, args.fn):
logging.info(f"finished conversion of {infn} to {outfn} (returned {rv})")
endtime = str(datetime.now())[:-7]
logging.info(f"completed at {endtime}.")
def setup():
"""Parse command-line arguments."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--log",
default="warning",
choices=["debug", "info", "warning", "error"],
help="logging level (defaults to 'warning')",
)
parser.add_argument("-v", "--version", action="version", version=__version__)
parser.add_argument(
"-l",
"--level",
action="store_true",
default=False,
help="Correct color levels (default: no)",
)
parser.add_argument(
"-q", "--quality", type=int, default=80, help="JPEG quailty level (default: 80)"
)
parser.add_argument(
"fn", nargs="*", metavar="filename", help="DICOM files to process"
)
args = parser.parse_args(sys.argv[1:])
logging.basicConfig(
level=getattr(logging, args.log.upper(), None),
format="%(levelname)s: %(message)s",
)
logging.debug(f"command line arguments = {sys.argv}")
logging.debug(f"parsed arguments = {args}")
# Check for requisites
try:
sp.run(["convert"], stdout=sp.DEVNULL, stderr=sp.DEVNULL)
logging.info("found “convert”")
except FileNotFoundError:
logging.error("the program “convert” cannot be found")
sys.exit(1)
return args
def convert(filename, quality, level):
"""
Convert a DICOM file to a JPEG file.
Removing the blank areas from the Philips detector.
Arguments:
filename: name of the file to convert.
quality: JPEG quality to apply
level: Boolean to indicate whether level adustment should be done.
Returns:
Tuple of (input filename, output filename, convert return value)
"""
outname = filename.strip() + ".jpg"
size = "1574x2048"
args = [
"convert",
filename,
"-units",
"PixelsPerInch",
"-density",
"300",
"-depth",
"8",
"-crop",
size + "+232+0",
"-page",
size + "+0+0",
"-auto-gamma",
"-quality",
str(quality),
]
if level:
args += ["-level", "-35%,70%,0.5"]
args.append(outname)
cp = sp.run(args, stdout=sp.DEVNULL, stderr=sp.DEVNULL)
return (filename, outname, cp.returncode)
if __name__ == "__main__":
main()
Alternatively, you can manage a bunch of subprocesses (in the form of Popen objects) directly, as shown below.
(This was older code, now modified for Python 3)
import os
import sys
import subprocess
from multiprocessing import cpu_count
from time import sleep
def checkfor(args):
"""Make sure that a program necessary for using this script is
available.
Arguments:
args -- string or list of strings of commands. A single string may
not contain spaces.
"""
if isinstance(args, str):
if " " in args:
raise ValueError("No spaces in single command allowed.")
args = [args]
try:
with open("/dev/null", "w") as bb:
subprocess.check_call(args, stdout=bb, stderr=bb)
except Exception:
print("Required program '{}' not found! exiting.".format(args[0]))
sys.exit(1)
def startconvert(fname):
"""Use the convert(1) program from the ImageMagick suite to convert the
image and crop it."""
size = "1574x2048"
args = [
"convert",
fname,
"-units",
"PixelsPerInch",
"-density",
"300",
"-crop",
size + "+232+0",
"-page",
size + "+0+0",
fname + ".png",
]
with open("/dev/null") as bb:
p = subprocess.Popen(args, stdout=bb, stderr=bb)
print("Start processing", fname)
return (fname, p)
def manageprocs(proclist):
"""Check a list of subprocesses for processes that have ended and
remove them from the list.
"""
for it in proclist:
fn, pr = it
result = pr.poll()
if result is not None:
proclist.remove(it)
if result == 0:
print("Finished processing", fn)
else:
s = "The conversion of {} exited with error code {}."
print(s.format(fn, result))
sleep(0.5)
def main(argv):
"""Main program.
Keyword arguments:
argv -- command line arguments
"""
if len(argv) == 1:
path, binary = os.path.split(argv[0])
print("Usage: {} [file ...]".format(binary))
sys.exit(0)
del argv[0] # delete the name of the script.
checkfor("convert")
procs = []
maxprocs = cpu_count()
for ifile in argv:
while len(procs) == maxprocs:
manageprocs(procs)
procs.append(startconvert(ifile))
while len(procs) > 0:
manageprocs(procs)
# This is the main program ##
if __name__ == "__main__":
main(sys.argv)

TypeError: exit expected at most 1 arguments, got 3

I made a Python script to compare files (compareRFRegion). I call this script from a Perl script:
$cmd_return = `python compareRFRegion.py -c Working/channels_US_full.ini -r RF_US902_full`;
But I get this error:
Traceback (most recent call last):
File "compareRFRegion.py", line 355, in <module>
input_filename, rf_region_filename)
File "compareRFRegion.py", line 88, in open_files
"!!! Check it's in the current directory or the path is correct")
TypeError: exit expected at most 1 arguments, got 3
Here's my Python script:
#!/usr/bin/python
import os
import sys
import re
import getopt
# Channel list from .ini
channel_list = []
# Channel list from RF_region.xml
rf_channel_list = []
# lgw list
lgw_list = []
rf_region_list = []
class Channel:
"""attributes
- index
- LC
- subband
- freqhz
- usedforrx2
"""
def __init__(self):
self.index = 0 # [channel:x]
#
self.LC = 0 # name=LCx
self.subband = 0 # subband=x
self.freqhz = 0 # freqhz=x
self.usedforrx2 = 0 # usedforrx2=x
self.bandwidth = 0 # bandwidth=x
self.power = 0 # power=x
def display_channel(self):
print("Channel #{} - LC{} - Subband = {} - Freq = {} - UsedForRX2 = {} - Power = {}\n".format(self.index,
self.LC,
self.subband,
self.freqhz,
self.usedforrx2,
self.power))
def __eq__(self, channel):
# if self.LC != channel.LC:
# print ("LC different : {} - {} ", self.LC, channel.LC)
# if self.subband != channel.subband:
# print ("Subband different : {} - {} ", self.subband, channel.subband)
# if self.freqhz != channel.freqhz:
# print ("FreqHz different : {} - {} ", self.freqhz, channel.freqhz)
# if self.usedforrx2 != channel.usedforrx2:
# print ("Usedforrx2 different : {} - {} ", self.usedforrx2, channel.usedforrx2)
# if self.power != channel.power:
# print ("Power different : {} - {} ", self.power, channel.power)
return self.LC == channel.LC and self.subband == channel.subband and self.freqhz == channel.freqhz and self.usedforrx2 == channel.usedforrx2 and self.power == channel.power
def __ne__(self, channel):
return not self.__eq__(channel)
# File handling
def open_files(input_filename, rf_region_filename):
input_file = None
lgw_file = None
if input_filename:
try:
input_file = open(input_filename, "r")
except IOError:
sys.exit("Could not open", input_filename,
"!!! Check it's in the current directory or the path is correct")
try:
rf_region_file = open(rf_region_filename, "r")
except IOError:
input_file.close()
sys.exit("Could not open", rf_region_filename,
"!!! Check it's in the current directory or the path is correct")
return input_file, rf_region_file
def close_files(input_file, rf_region_file):
input_file.close()
rf_region_file.close()
# Read script arguments
def read_param(argv):
channel_filename = ''
rf_region_filename = ''
lgw_filename = ''
try:
opts, args = getopt.getopt(argv, "hc:l:r:")
except getopt.GetoptError:
print('compareRFRegion.py -c <channel_file> -r <RF_region_file>')
print('compareRFRegion.py -l <lgw_file> -r <RF_region_file>')
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print('compareRFRegion.py -c <channel_file> -r <RF_region_file>')
print('compareRFRegion.py -l <lgw_file> -r <RF_region_file>')
sys.exit()
elif opt in ("-c"):
channel_filename = arg
elif opt in ("-l"):
lgw_filename = arg
elif opt in ("-r"):
rf_region_filename = arg
# print('Channel file is "', channel_filename)
# print('RF_region file is "', rf_region_filename)
return channel_filename, lgw_filename, rf_region_filename
# process channel from RF_region.xml
def process_rf_channel(match, channel):
global rf_channel_list
if channel is not None:
rf_channel_list.append(channel)
channel = Channel()
return channel
def process_rx2_LC(match, channel):
global rf_channel_list
if channel is not None:
rf_channel_list.append(channel)
channel = Channel()
channel.LC = int(match.group(1))
channel.usedforrx2 = 1
return channel
def process_rf_freqhz(match, channel):
channel.freqhz = int(float(match.group(1)) * 1000000)
return channel
# process channel from channels.ini
def process_channel(match, channel):
global channel_list
# we store the previous channel in channel_list (except the first one)
if channel is not None:
channel_list.append(channel)
channel = Channel()
channel.index = int(match.group(1))
return channel
# processes for all files
def process_LC(match, channel):
channel.LC = int(match.group(1))
return channel
def process_subband(match, channel):
channel.subband = int(match.group(1))
return channel
def process_freqhz(match, channel):
channel.freqhz = int(match.group(1))
return channel
def process_usedforrx2(match, channel):
channel.usedforrx2 = int(match.group(1))
return channel
def process_bandwidth(match, channel):
channel.bandwidth = int(match.group(1))
return channel
def process_power(match, channel):
channel.power = int(match.group(1))
return channel
# Read functions
def read_channels(channel_file):
global channel_list
actions = ((r"\[channel:(\d+)\]", process_channel),
(r"name=LC((d)?.+)", process_LC),
(r"subband=(\d+)", process_subband),
(r"freqhz=(\d+\.\d+)", process_rf_freqhz),
(r"usedforrx2=([0|1])", process_usedforrx2),
(r"bandwidth=\$\{BW_(\d+)KHZ\}", process_bandwidth),
(r"power=(\d+)", process_power))
channel = None
for line in channel_file:
# print(line)
for regex, action in actions:
match = re.search(regex, line)
if match:
channel = action(match, channel)
break
# append the last channel in list
if channel is not None:
channel_list.append(channel)
def read_rf_region(rf_region_file):
global rf_channel_list
actions = ((r"<[RT]xChannel>", process_rf_channel),
(r"<LC>(\d+)<\/LC>", process_LC),
(r"<SB>(\d+)<\/SB>", process_subband),
(r"<Frequency>(\d+\.\d+)<\/Frequency>", process_rf_freqhz),
(r"<UsedForRX2>([0|1])<\/UsedForRX2>", process_usedforrx2),
(r"<Bandwidth>(\d+)<\/Bandwidth>", process_bandwidth),
(r"<RX2LC>(\d+)<\/RX2LC>", process_rx2_LC),
(r"<RX2SB>(\d+)<\/RX2SB>", process_subband),
(r"<RX2Freq>(\d+\.\d+)<\/RX2Freq>", process_rf_freqhz),
(r"<RX2TxPower>(\d+)<\/RX2TxPower>", process_power))
channel = None
for line in rf_region_file:
# print(line)
for regex, action in actions:
match = re.search(regex, line)
if match:
channel = action(match, channel)
break
# append the last channel in list
if channel is not None:
rf_channel_list.append(channel)
def read_rf_region_lgw(rf_region_file):
global rf_region_list
regexs = (r"<RFRegionId>(.+)<\/RFRegionId>",
r"<LRR_power>(\d+)<\/LRR_power>")
for line in rf_region_file:
# print(line)
for regex in regexs:
match = re.search(regex, line)
if match:
rf_region_list.append(match.group(1))
break
def read_lgw(lgw_file):
regexs = (r"rfregionid=(.+)", r"power=(\d+)")
global lgw_list
for line in lgw_file:
# print(line)
for regex in regexs:
match = re.search(regex, line)
if match:
lgw_list.append(match.group(1))
break
# Compare functions
def compareChannels():
for channel, rf_channel in zip(channel_list, rf_channel_list):
if channel != rf_channel:
# channel.display_channel()
# rf_channel.display_channel()
print(0)
return
print(1)
def compareLgw():
for lgw_param, rf_region_param in zip(lgw_list, rf_region_list):
if lgw_param != rf_region_param:
# print(lgw_param)
# print(rf_region_param)
print(0)
return
print(1)
# def move_rx2_channel():
# for i, channel in enumerate(rf_channel_list):
# if channel.usedforrx2 == 1:
# tmp = rf_channel_list.pop(i)
# rf_channel_list.append(tmp)
# return
#if __name__ == "__main__":
channel_filename, lgw_filename, rf_region_filename = read_param(sys.argv[
1:])
input_filename = ''
input_file = None
isChannelType = True
if channel_filename:
input_filename = channel_filename
elif lgw_filename:
input_filename = lgw_filename
isChannelType = False
input_file, rf_region_file = open_files(
input_filename, rf_region_filename)
# move_rx2_channel()
if isChannelType:
read_rf_region(rf_region_file)
read_channels(input_file)
compareChannels()
else:
read_rf_region_lgw(rf_region_file)
read_lgw(input_file)
compareLgw()
# print("List size is", len(channel_list))
# print("List rf size is", len(rf_channel_list))
# for channel, rf_channel in zip(channel_list, rf_channel_list):
# channel.display_channel()
# rf_channel.display_channel()
close_files(input_file, rf_region_file)
I am able to execute this in standalone in linux terminal by adding if __name__ == "__main__": (commented here). It works fine. But not by calling it from Perl. Maybe there is something I am missing about calling a Python script from Perl ?

instead of
sys.exit("Could not open", rf_region_filename,
"!!! Check it's in the current directory or the path is correct")
try
sys.exit("Could not open" + rf_region_filename + \
"!!! Check it's in the current directory or the path is correct")
adding in the commas isn't like the print statement, and doesn't concatenate them. Instead it treats the three strings as 3 different arguments. Adding in the additions signs will concatenate them.

Please, see - https://docs.python.org/3/library/sys.html#sys.exit
Your call of sys.exit() with 3 args is wrong, expected only one - exit code (optional)

You twice call sys.exit with two many arguments (as the error tells you :) )
sys.exit("Could not open", input_filename,
"!!! Check it's in the current directory or the path is correct")
change to e.g.
print("Could not open", input_filename,
"!!! Check it's in the current directory or the path is correct")
sys.exit(1)

You need to read your exception. It says that you are calling sys.exit() with 3 arguments, but only 1 is allowed.
Read the docs on sys.exit([arg])
And here is notes on optional argument arg:
The optional argument arg can be an integer giving the exit status (defaulting to zero), or another type of object. If it is an integer, zero is considered “successful termination” and any nonzero value is considered “abnormal termination” by shells and the like. Most systems require it to be in the range 0–127, and produce undefined results otherwise. Some systems have a convention for assigning specific meanings to specific exit codes, but these are generally underdeveloped; Unix programs generally use 2 for command line syntax errors and 1 for all other kind of errors. If another type of object is passed, None is equivalent to passing zero, and any other object is printed to stderr and results in an exit code of 1. In particular, sys.exit("some error message") is a quick way to exit a program when an error occurs.
So you need to refactor your code, for sys.exit to get only one argument.

Python threads repeating values

I am trying to write a python SHA512 brute forcer.
I use a Queue to store the values in the wordlist and then compare them against the encrypted hash.
The problem is that, instead of the values being popped out of the Queue, they are reused by other threads. So basically, instead of having the whole work split between threads to make things faster, I got several threads doing the exact same thing. How can I fix this?
I want something like this: https://github.com/WillPennell/Python/blob/master/Black-Hat-Python/BHP-Code/Chapter5/content_bruter.py#L20
import threading
import thread
import Queue
import os,sys
import crypt
import codecs
from datetime import datetime,timedelta
import argparse
today = datetime.today()
resume = None
threads = 5
def build_wordlist(wordlist_file):
fd = open(wordlist_file,"rb")
raw_words = fd.readlines()
fd.close()
found_resume = False
words = Queue.Queue()
for word in raw_words:
word = word.rstrip()
if resume is not None:
if found_resume:
words.put(word)
else:
if word == resume:
found_resume = True
print "Resuming wordlist from: %s" % resume
else:
words.put(word)
return words
def testPass(cryptPass,user):
word_queue = build_wordlist('test.txt')
while not word_queue.empty():
attempt = word_queue.get()
ctype = cryptPass.split("$")[1]
if ctype == '6':
print "[+] Hash type SHA-512 detected ..."
salt = cryptPass.split("$")[2]
insalt = "$" + ctype + "$" + salt + "$"
word = attempt
cryptWord = crypt.crypt(word,insalt)
if (cryptWord == cryptPass):
time = time = str(datetime.today() - today)
print "[+] Found password for the user: " + user + " ====> " + word + " Time: "+time+"\n"
return
print "Password not found for the user: " + user
print "Moving on to next user..."
exit
def main():
parse = argparse.ArgumentParser(description='A simple brute force /etc/shadow .')
parse.add_argument('-f', action='store', dest='path', help='Path to shadow file, example: \'/etc/shadow\'')
argus=parse.parse_args()
if argus.path == None:
parse.print_help()
exit
else:
build_wordlist('test.txt')
passFile = open (argus.path,'r')
for line in passFile.readlines():
line = line.replace("\n","").split(":")
if not line[1] in [ 'x' , '*' , '!' ]:
user = line[0]
cryptPass = line[1]
for i in range(threads):
t = threading.Thread(target=testPass,args=(cryptPass,user))
t.daemon = True
t.start()
if __name__=="__main__":
main()
EDIT: I realized there are 2 ways I can do this:
first, I can create a thread for each user, which is not what I want.
Second, I can split the work of each user through several threads, which is what I want.

Let's look at this block of code :
for i in range(threads):
t = threading.Thread(target=testPass,args=(cryptPass,user))
t.daemon = True
t.start()
And let's describe what this is doing for each thread you start :
create a new Queue object from test.txt as defined by build_wordlist
Process the queue from step 1
It sounds like your desired behavior is to multithread some processing step on a single queue rather than create duplicates of the same queue. So this means your "testPass" method should probably take in a Queue object. i.e.
q = build_wordlist('test.txt')
for i in range(threads):
t = threading.Thread(target=testPass,args=(q, cryptPass,user))
t.daemon = True
t.start()
and testPass should look like :
def testPass(queue, cryptPass, user):
word_queue = queue
... stuff ...

Suspected thread issue with pyinotify

I have been working with pyinotify and I am having issues with it where after multiple changes to a folder it simply stops receiving notifications. I have a feeling it is something to do with the the fact that two threads are running; namely the notifier thread and the wxpython thread.
The purpose of the app is to essentially load a picture to the screen when it detects an ip connection, monitor a folder for the file 'Checklist' and based on that file do some processing i.e move files around.
It works intermittently but being a python newbie Im not exactly sure what the issue might be as I have basically taken the threaded example and worked around it. Sometimes, it only gets one notification and stops receiving file change notifications.
Additionally, if I restart the linux box and try again, it works for a good number of file changes and then stops receiving notifications again which makes me think that perhaps its not releasing the watches properly?
Any help would be greatly appreciated as well as optimizations and improvements are very welcome. I'm sure I could learn a lot from the feedback. The code is below
import pyinotify
import os.path
import shutil
import errno
import subprocess
import logging
import wx
import time
import signal
import sys
#update CHECKLIST name
CHECKLIST = 'Checklist' #this must exist in the update archive
#static values
DIR_UPDATE = 'd'
FILE_UPDATE = 'f'
PING_IP = ' localhost' # change in production
#paths
WATCH_PATH = '/home/test'
LOG_PATH = '/home/test/update.log'
CONNECTED_IMG = 'update.jpg'
UPDATING_IMG = 'updating.png'
#msgs
UPDATEFOUND_MSG = ' Update Found '
UPDATEUNZIPPEDSTART_MSG = ' Update unzipping '
UPDATEUNZIPPED_MSG = ' Update unzipped '
UPDATEFILE_MSG = ' Update file '
UPDATEFILEMSG_CONT = ' moved into path '
REMOVEFILEMSG_CONT = ' removed from update folder '
UPDATECOMPLETE_CONT = ' Update complete'
ROADANGELRESTART_MSG = ' Update restarting app '
DIRCREATED_MSG = ' Directory created at path '
#errors
UPDATEFAILED_MSG = ' Update process failed on '
BADLYFORMED_MSG = ' Badly formed src/dest combination '
UPDATESRCUNAVAILABLE = ' Invalid update file specified '
UPDATEDESTUNAVAILABLE = ' Invalid update destination specified '
INVALIDUPDATEFORMAT = ' Invalid format string '
#on startup create the watchfolder if it doesnt exist
WM = pyinotify.WatchManager() # Watch Manager
WM_MASK = pyinotify.IN_CLOSE_WRITE # watched events
#setup logger
LOGGER = logging.getLogger('Updater')
LOG_HANDLE = logging.FileHandler(LOG_PATH)
FORMATTER = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
LOG_HANDLE.setFormatter(FORMATTER)
LOGGER.addHandler(LOG_HANDLE)
LOGGER.setLevel(logging.INFO)
#Global values used primarily in the main function loop
HANDLER = None
NOTIFIER = None
WDD = None
UPDATE_UI = None
WINDOW = None
CURR_IMG = None
LAST_CURRIMG = None
class EventHandler(pyinotify.ProcessEvent):
VERBOSE = False
""" Main class to monitor file events and process accordingly"""
def process_IN_CLOSE_WRITE(self, event):
""" Only executes when a Checklist has finished being written to"""
path = event.pathname
print 'evt'
#look for the update_ready file before processing
if (os.path.basename(path) == 'Checklist'):
EventHandler.parse_updates(WATCH_PATH)
global CURR_IMG
CURR_IMG = os.path.join(WATCH_PATH, UPDATING_IMG)
show_window()
print 'update completed'
time.sleep(1000)
#classmethod
def parse_updates(cls, path):
""" parses update files """
#handle errors for opening the file
file_path = os.path.join(path, CHECKLIST)
print file_path
files = open(file_path)
#handle errors for malformed tuples-done
#handle errors for unavailable files-done
#handle errors for unavailable dests-done #created automatically
#handle permission errors
for line in files:
#remove linebreaks etc and ensure its not empty
if line.strip():
array = line.split('=')
length = len(array)
if length == 3:
EventHandler.process_line(path, array)
else:
if length > 0:
EventHandler.print_bad_msg(array[0])
else:
EventHandler.print_bad_msg()
print 'removing ', file_path
os.remove(file_path) #remove the checklist file
#classmethod
def mkdir(cls, path):
""" makes a directory from a path"""
try:
os.mkdir(path)
print DIRCREATED_MSG, path
except OSError, err:
print err
if err.errno != errno.EEXIST: #thrown when the dir already exists
return False
#classmethod
def move_file(cls, src, dest):
""" moves a file from src to dest and remove after
expects that the dest already exists at this point
otherwise ignores the move"""
#print 'moving from', src, 'to ', dest
if os.path.isfile(dest):
shutil.copy2(src, dest)
else:
print UPDATEDESTUNAVAILABLE
#remove the src file when done
os.remove(src)
#classmethod
def process_line(cls, path, array):
""" process a line from the checklist"""
#remove newlines etc
update_file = array[0].strip()
update_src = os.path.join(path, update_file)
update_dest = array[1].strip()
update_type = array[2].strip()
#ensure we have valid values in all three fields
if update_file and update_dest and update_type:
#ensure the src file exists
if os.path.isfile(update_src):
#check if destination is directory and
#copy the file into the directory
if update_type == DIR_UPDATE:
EventHandler.mkdir(update_dest)
dest = os.path.join(update_dest, update_file)
EventHandler.move_file(update_src, dest)
else:
EventHandler.move_file(update_src, update_dest)
else:
print UPDATESRCUNAVAILABLE
else:
print INVALIDUPDATEFORMAT
#classmethod
def print_bad_msg(cls, msg = ''):
""" print a badly formed message with optional value"""
if msg:
print BADLYFORMED_MSG, msg
else:
print BADLYFORMED_MSG
class UpdateFrame(wx.Frame):
""" Displays update images to screen"""
def __init__(self, path):
wx.Frame.__init__(self, None, wx.ID_ANY)
image_file = path
image = wx.Bitmap(image_file)
image_size = image.GetSize()
# set the frame size to fit the screen size
self.SetClientSize(wx.DisplaySize())
# bitmap's upper left corner is in frame position (x, y)
# by default pos=(0, 0)
wx.StaticBitmap(self, wx.ID_ANY, image, size = image_size)
# the parent is the frame
self.SetTitle('Update Mode')
def ping_ip():
""" ping once to establish connection """
ret = subprocess.call("ping -c 1 %s" % PING_IP,
shell=True,
stdout=open('/dev/null', 'w'),
stderr=subprocess.STDOUT)
if ret == 0:
return True
else:
return False
def show_window():
""" update screen window when currimage changes is set """
global UPDATE_UI
global WINDOW
global CURR_IMG
global LAST_CURRIMG
if LAST_CURRIMG != CURR_IMG:
if not UPDATE_UI:
UPDATE_UI = wx.App()
if not WINDOW:
WINDOW = UpdateFrame(CURR_IMG)
UPDATE_UI.ExitMainLoop()
while(UPDATE_UI.IsMainLoopRunning()):
pass
WINDOW.Destroy()
WINDOW = UpdateFrame(CURR_IMG)
WINDOW.Show(True)
UPDATE_UI.MainLoop()
LAST_CURRIMG = CURR_IMG
print 'changed'
def in_updatemode():
return ping_ip()
while True:
try:
if not in_updatemode():
print 'waiting for connect'
time.sleep(3)
if NOTIFIER:
NOTIFIER.stop()
else:
if not HANDLER:
HANDLER = EventHandler()
if not NOTIFIER:
NOTIFIER = pyinotify.ThreadedNotifier(WM, HANDLER)
NOTIFIER.start()
if not WDD:
WDD = WM.add_watch(WATCH_PATH, WM_MASK, rec=True,quiet=False)
# ip is active so show the image and start the notifier
# state = ip active
CURR_IMG = os.path.join(WATCH_PATH, CONNECTED_IMG)
show_window()
print 'here'
except KeyboardInterrupt:
print 'out'
NOTIFIER.stop()
break

I basically found out that that the issue was indeed the fact that pyinotify's threaded notifier and wxPython's main loop dont play nice together.
The solution was to create a custom main loop (something I didn't know you could do in the first place) and place the pyinotify's threaded notifier within that loop. That way it ran as part of the wxPython main loop.
I got the idea from
http://www.java2s.com/Open-Source/Python/GUI/wxPython/wxPython-src-2.8.11.0/wxPython/samples/mainloop/mainloop.py.htm
Code below explains the concept
class CustomApp(wx.App):
def MainLoop(self):
global HANDLER
global WM
global NOTIFIER
global WDD
global UPDATE_UI
global PING_TIMER
# Create an event loop and make it active. If you are
# only going to temporarily have a nested event loop then
# you should get a reference to the old one and set it as
# the active event loop when you are done with this one...
evtloop = wx.EventLoop()
old = wx.EventLoop.GetActive()
wx.EventLoop.SetActive(evtloop)
# This outer loop determines when to exit the application,
# for this example we let the main frame reset this flag
# when it closes.
while self.keepGoing:
# At this point in the outer loop you could do
# whatever you implemented your own MainLoop for. It
# should be quick and non-blocking, otherwise your GUI
# will freeze.
# call_your_code_here()
if not HANDLER:
HANDLER = EventHandler()
if not WM:
WM = pyinotify.WatchManager() # Watch Manager
if not NOTIFIER:
NOTIFIER = pyinotify.ThreadedNotifier(WM, HANDLER)
NOTIFIER.start()
print 'notifier started'
if not WDD:
WDD = WM.add_watch(WATCH_PATH, WM_MASK, rec=True,quiet=False)
# This inner loop will process any GUI events
# until there are no more waiting.
while evtloop.Pending():
evtloop.Dispatch()
# Send idle events to idle handlers. You may want to
# throttle this back a bit somehow so there is not too
# much CPU time spent in the idle handlers. For this
# example, I'll just snooze a little...
time.sleep(0.10)
self.ProcessIdle()
wx.EventLoop.SetActive(old)
def OnInit(self):
global UPDATE_UI
if not UPDATE_UI:
UPDATE_UI = Updater()
UPDATE_UI.Show()
self.SetTopWindow(UPDATE_UI)
self.keepGoing = True
return True
"--------------------------------------------------------------------------------------"
Watcher()
app = CustomApp(False)
Additionally, I wanted to catch SIGINT in the program and I solved it using the recipe from this url
http://code.activestate.com/recipes/496735-workaround-for-missed-sigint-in-multithreaded-prog/
Hope this helps another python newbie or oldie :)

Why wont my Simple CGI Server handle AJAX Post requests?

I have created my own CGI python server script (that serves on port 8000) by following a tutorial. The server works beautifully if I want to generate web pages from python scripts, or serve a native HTML page BUT it doesn't work for when I make an AJAX POST request?
If I make an AJAX request to the python file aaa.py (using the javascript below) my server prints out the following error text:
Code 501, message can only POST to to CGI scripts
"POST /aaa.py HTTP/1.1" 501 -
What do you think I need to do to allow my python cgi server to allow/handle AJAX requests?
My CGI server:
__version__ = "0.4"
__all__ = ["CGIHTTPRequestHandler"]
import os
import sys
import urllib
import BaseHTTPServer
import SimpleHTTPServer
import select
class CGIHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
"""Complete HTTP server with GET, HEAD and POST commands.
GET and HEAD also support running CGI scripts.
The POST command is *only* implemented for CGI scripts.
"""
# Determine platform specifics
have_fork = hasattr(os, 'fork')
have_popen2 = hasattr(os, 'popen2')
have_popen3 = hasattr(os, 'popen3')
# pretend we don't have these to force execution in process
have_fork = 0
# Make rfile unbuffered -- we need to read one line and then pass
# the rest to a subprocess, so we can't use buffered input.
rbufsize = 0
def do_POST(self):
"""Serve a POST request.
This is only implemented for CGI scripts.
"""
if self.is_cgi():
self.run_cgi()
else:
self.send_error(501, "Can only POST to CGI scripts")
def send_head(self):
"""Version of send_head that support CGI scripts"""
if self.is_cgi():
return self.run_cgi()
else:
return SimpleHTTPServer.SimpleHTTPRequestHandler.send_head(self)
def is_cgi(self):
"""Test whether self.path corresponds to a CGI script.
Return a tuple (dir, rest) if self.path requires running a
CGI script, None if not. Note that rest begins with a
slash if it is not empty.
The default implementation tests whether the path
begins with one of the strings in the list
self.cgi_directories (and the next character is a '/'
or the end of the string).
"""
path = self.path
for x in self.cgi_directories:
i = len(x)
if path[:i] == x and (not path[i:] or path[i] == '/'):
self.cgi_info = path[:i], path[i+1:]
return True
return False
cgi_directories = ['/cgi-bin', '/htbin']
def is_executable(self, path):
"""Test whether argument path is an executable file."""
return executable(path)
def is_python(self, path):
"""Test whether argument path is a Python script."""
head, tail = os.path.splitext(path)
return tail.lower() in (".py", ".pyw")
def run_cgi(self):
"""Execute a CGI script."""
dir, rest = self.cgi_info
i = rest.rfind('?')
if i >= 0:
rest, query = rest[:i], rest[i+1:]
else:
query = ''
# i = rest.find('/')
# if i >= 0:
# script, rest = rest[:i], rest[i:]
# else:
# script, rest = rest, ''
script = rest
scriptname = dir + '/' + script
scriptfile = self.translate_path(scriptname)
if not os.path.exists(scriptfile):
self.send_error(404, "No such CGI script (%s)" % `scriptname`)
return
if not os.path.isfile(scriptfile):
self.send_error(403, "CGI script is not a plain file (%s)" %
`scriptname`)
return
ispy = self.is_python(scriptname)
if not ispy:
if not (self.have_fork or self.have_popen2 or self.have_popen3):
self.send_error(403, "CGI script is not a Python script (%s)" %
`scriptname`)
return
if not self.is_executable(scriptfile):
self.send_error(403, "CGI script is not executable (%s)" %
`scriptname`)
return
# Reference: http://hoohoo.ncsa.uiuc.edu/cgi/env.html
# XXX Much of the following could be prepared ahead of time!
env = {}
env['SERVER_SOFTWARE'] = self.version_string()
env['SERVER_NAME'] = self.server.server_name
env['GATEWAY_INTERFACE'] = 'CGI/1.1'
env['SERVER_PROTOCOL'] = self.protocol_version
env['SERVER_PORT'] = str(self.server.server_port)
env['REQUEST_METHOD'] = self.command
uqrest = urllib.unquote(rest)
env['PATH_INFO'] = uqrest
env['PATH_TRANSLATED'] = self.translate_path(uqrest)
env['SCRIPT_NAME'] = scriptname
if query:
env['QUERY_STRING'] = query
host = self.address_string()
if host != self.client_address[0]:
env['REMOTE_HOST'] = host
env['REMOTE_ADDR'] = self.client_address[0]
# XXX AUTH_TYPE
# XXX REMOTE_USER
# XXX REMOTE_IDENT
if self.headers.typeheader is None:
env['CONTENT_TYPE'] = self.headers.type
else:
env['CONTENT_TYPE'] = self.headers.typeheader
length = self.headers.getheader('content-length')
if length:
env['CONTENT_LENGTH'] = length
accept = []
for line in self.headers.getallmatchingheaders('accept'):
if line[:1] in "\t\n\r ":
accept.append(line.strip())
else:
accept = accept + line[7:].split(',')
env['HTTP_ACCEPT'] = ','.join(accept)
ua = self.headers.getheader('user-agent')
if ua:
env['HTTP_USER_AGENT'] = ua
co = filter(None, self.headers.getheaders('cookie'))
if co:
env['HTTP_COOKIE'] = ', '.join(co)
# XXX Other HTTP_* headers
if not self.have_fork:
# Since we're setting the env in the parent, provide empty
# values to override previously set values
for k in ('QUERY_STRING', 'REMOTE_HOST', 'CONTENT_LENGTH',
'HTTP_USER_AGENT'):
###, 'HTTP_COOKIE' -- removed by S.
env.setdefault(k, "")
# for key in env.keys():
# print key + " '" + env[key] + "'"
os.environ.update(env)
self.send_response(200, "Script output follows")
decoded_query = query.replace('+', ' ')
if self.have_fork:
# Unix -- fork as we should
args = [script]
if '=' not in decoded_query:
args.append(decoded_query)
nobody = nobody_uid()
self.wfile.flush() # Always flush before forking
pid = os.fork()
if pid != 0:
# Parent
pid, sts = os.waitpid(pid, 0)
# throw away additional data [see bug #427345]
while select.select([self.rfile], [], [], 0)[0]:
if not self.rfile.read(1):
break
if sts:
self.log_error("CGI script exit status %#x", sts)
return
# Child
try:
try:
os.setuid(nobody)
except os.error:
pass
os.dup2(self.rfile.fileno(), 0)
os.dup2(self.wfile.fileno(), 1)
os.execve(scriptfile, args, os.environ)
except:
self.server.handle_error(self.request, self.client_address)
os._exit(127)
elif self.have_popen2 or self.have_popen3:
# Windows -- use popen2 or popen3 to create a subprocess
import shutil
if self.have_popen3:
popenx = os.popen3
else:
popenx = os.popen2
cmdline = scriptfile
if self.is_python(scriptfile):
interp = sys.executable
if interp.lower().endswith("w.exe"):
# On Windows, use python.exe, not pythonw.exe
interp = interp[:-5] + interp[-4:]
cmdline = "%s -u \"%s\"" % (interp, cmdline)
if '=' not in query and '"' not in query:
cmdline = '%s "%s"' % (cmdline, query)
self.log_message("command: %s", cmdline)
try:
nbytes = int(length)
except (TypeError, ValueError):
nbytes = 0
files = popenx(cmdline, 'b')
fi = files[0]
fo = files[1]
if self.have_popen3:
fe = files[2]
if self.command.lower() == "post" and nbytes > 0:
data = self.rfile.read(nbytes)
fi.write(data)
# throw away additional data [see bug #427345]
while select.select([self.rfile._sock], [], [], 0)[0]:
if not self.rfile._sock.recv(1):
break
fi.close()
shutil.copyfileobj(fo, self.wfile)
if self.have_popen3:
errors = fe.read()
fe.close()
if errors:
self.log_error('%s', errors)
sts = fo.close()
if sts:
self.log_error("CGI script exit status %#x", sts)
else:
self.log_message("CGI script exited OK")
else:
# Other O.S. -- execute script in this process
save_argv = sys.argv
save_stdin = sys.stdin
save_stdout = sys.stdout
save_stderr = sys.stderr
try:
try:
sys.argv = [scriptfile]
if '=' not in decoded_query:
sys.argv.append(decoded_query)
sys.stdout = self.wfile
sys.stdin = self.rfile
execfile(scriptfile, {"__name__": "__main__"})
finally:
sys.argv = save_argv
sys.stdin = save_stdin
sys.stdout = save_stdout
sys.stderr = save_stderr
except SystemExit, sts:
self.log_error("CGI script exit status %s", str(sts))
else:
self.log_message("CGI script exited OK")
nobody = None
def nobody_uid():
"""Internal routine to get nobody's uid"""
global nobody
if nobody:
return nobody
try:
import pwd
except ImportError:
return -1
try:
nobody = pwd.getpwnam('nobody')[2]
except KeyError:
nobody = 1 + max(map(lambda x: x[2], pwd.getpwall()))
return nobody
def executable(path):
"""Test for executable file."""
try:
st = os.stat(path)
except os.error:
return False
return st.st_mode & 0111 != 0
def test(HandlerClass = CGIHTTPRequestHandler,
ServerClass = BaseHTTPServer.HTTPServer):
SimpleHTTPServer.test(HandlerClass, ServerClass)
if __name__ == '__main__':
test()
Code in aaa.py:
#!/usr/bin/env python
import cgitb; cgitb.enable()
import cgi
import os
print "Content-Type: text/html\n"
input_data = cgi.FieldStorage()
print "hello"
My AJAX/ Javascript:
function onTest( dest, params )
{
var xmlhttp;
if (window.XMLHttpRequest)
{// code for IE7+, Firefox, Chrome, Opera, Safari
xmlhttp=new XMLHttpRequest();
}
else
{// code for IE6, IE5
xmlhttp=new ActiveXObject("Microsoft.XMLHTTP");
}
xmlhttp.onreadystatechange=function()
{
if (xmlhttp.readyState==4 && xmlhttp.status==200)
{
document.getElementById( "bb" ).innerHTML = xmlhttp.responseText;
}
}
xmlhttp.open("POST",dest,true);
xmlhttp.setRequestHeader("Content-type","application/x-www-form-urlencoded");
xmlhttp.send( params );
}

when you use the python http cgi server, cgi scripts have to be under the subdir (from you server script ) /cgi-bin