I have been using a code to deal with some data in ~9000 files, since every single process takes some time, I managed to use mpi4py to speed it up. But I cannot make the program catch the ctrl+c signal and save the results already calculated. Here's the code:
import os
import sys
from string import atof
import gc
import pandas as pd
import numpy as np
from StructFunc.Structure_Function import SF_true, SF_fit_params
import scipy.io as sio
import mpi4py
from mpi4py import MPI
import time
import signal
# def handler(signal_num, frame):
# combine_list = comm.gather(p, root=0)
# if comm_rank == 0:
# print combine_list
# combine_dict = {}
# for sub_dict in combine_list:
# for cur_key in sub_dict.keys():
# combine_dict[cur_key] = sub_dict[cur_key]
# print combine_dict
# sio.savemat('./all_S82_DRW_params.mat', combine_dict)
# print "results before the interrupt has been saved."
# sys.exit()
# signal.signal(signal.SIGINT, handler)
comm = MPI.COMM_WORLD
comm_rank = comm.Get_rank()
comm_size = comm.Get_size()
bad_value = [-1, -99.99, -1]
lc_path = '/Users/zhanghaowen/Desktop/AGN/BroadBand_RM/QSO_S82'
model="DRW"
if comm_rank == 0:
file_list = os.listdir(lc_path)
# print file_list
sys.stderr.write('%d files were to be processed.' %len(file_list))
file_list = comm.bcast(file_list if comm_rank == 0 else None, root=0)
num_files = len(file_list)
# num_files = 6
local_files_offset = np.linspace(0, num_files, comm_size+1).astype('int')
local_files = file_list[local_files_offset[comm_rank] : local_files_offset[comm_rank + 1]]
sys.stderr.write('%d/%d processor gets %d/%d data' %(comm_rank, comm_size, len(local_files), num_files))
cnt = 0
p = {}
for ind, lc in enumerate(local_files):
# beginning of process
try:
print local_files[ind]
lc_file = open(os.path.join(lc_path, local_files[ind]), 'r')
lc_data = pd.read_csv(lc_file, header=None, names=['MJD', 'mag', 'err'], usecols=[0, 1, 2], sep=' ')
lc_file.close()
#remove the bad values in the data
lc_data['MJD'] = lc_data['MJD'].replace(to_replace=bad_value[0], value=np.nan)
lc_data['MJD'] = lc_data['MJD'].replace(to_replace=np.nan, value=np.nanmean(lc_data['MJD']))
lc_data['mag'] = lc_data['mag'].replace(to_replace=bad_value[1], value=np.nan)
lc_data['mag'] = lc_data['mag'].replace(to_replace=np.nan, value=np.nanmean(lc_data['mag']))
lc_data['err'] = lc_data['err'].replace(to_replace=bad_value[2], value=np.nan)
lc_data['err'] = lc_data['err'].replace(to_replace=np.nan, value=np.nanmean(lc_data['err']))
MJD = np.array(lc_data['MJD'])
mag = np.array(lc_data['mag'])
err = np.array(lc_data['err'])
SF_params = []
resamp_tag = 0
while resamp_tag < 1:
sim_err = np.array([abs(np.random.normal(0, err[i], size=1)) for i in range(len(err))]).reshape((1, len(err)))[0]
try:
p[lc] = SF_fit_params(MJD, mag, sim_err, MCMC_step=100, MCMC_threads=1, model=model)
cnt += 1
sys.stderr.write('processor %d has processed %d/%d files \n' %(comm_rank, cnt, len(local_files)))
print "finished the MCMC for %s \n" %lc
resamp_tag += 1
except:
continue
# end of process
except KeyboardInterrupt:
combine_list = comm.gather(p, root=0)
if comm_rank == 0:
print combine_list
combine_dict = {}
for sub_dict in combine_list:
for cur_key in sub_dict.keys():
combine_dict[cur_key] = sub_dict[cur_key]
print combine_dict
sio.savemat('./all_S82_DRW_params.mat', combine_dict)
print "save the dict."
os._exit()
combine_list = comm.gather(p, root=0)
if comm_rank == 0:
print combine_list
combine_dict = {}
for sub_dict in combine_list:
for cur_key in sub_dict.keys():
combine_dict[cur_key] = sub_dict[cur_key]
print combine_dict
sio.savemat('./all_S82_DRW_params.mat', combine_dict)
I have tried two ways to catch the ctrl+c signal, i.e. the handler function I defined but commented and the except KeyboardInterrupt trick. When I run the code using python all.py and then type ctrl+c, the script would catch the signal but keeps running after saving the results; while when I use mpirun -np 2 all.py and type ctrl+c, the script just stops without saving.
I think the problem under MPI mode may be that only the manager process catched the signal, but why didn't the single process stop in the non-MPI mode after I typed ctrl+c? Do anyone know how to make the worker process catch the signal and do something before terminating?
By the way I'm using openmpi.
Related
I am trying to use ZeroMQ for multiprocessing. I want to stream files from a tar file so I used the streamer.
Below is an instance of what want to do.
import time
import zmq
from zmq.devices.basedevice import ProcessDevice
from multiprocessing import Process
def server(frontend_port, number_of_workers):
context = zmq.Context()
socket = context.socket(zmq.PUSH)
socket.connect("tcp://127.0.0.1:%d" % frontend_port)
for i in range(0,10):
socket.send_json('#%s' % i)
for i in range(number_of_workers):
socket.send_json('STOP')
return True
def worker(work_num, backend_port):
context = zmq.Context()
socket = context.socket(zmq.PULL)
socket.connect("tcp://127.0.0.1:%d" % backend_port)
while True:
message = socket.recv_json()
if message == 'STOP':
break
print("Worker #%s got message! %s" % (work_num, message))
time.sleep(1)
def main():
frontend_port = 7559
backend_port = 7560
number_of_workers = 2
streamerdevice = ProcessDevice(zmq.STREAMER, zmq.PULL, zmq.PUSH)
streamerdevice.bind_in("tcp://127.0.0.1:%d" % frontend_port )
streamerdevice.bind_out("tcp://127.0.0.1:%d" % backend_port)
streamerdevice.setsockopt_in(zmq.IDENTITY, b'PULL')
streamerdevice.setsockopt_out(zmq.IDENTITY, b'PUSH')
streamerdevice.start()
processes = []
for work_num in range(number_of_workers):
w = Process(target=worker, args=(work_num,backend_port))
processes.append(w)
w.start()
time.sleep(1)
s = Process(target=server, args=(frontend_port,number_of_workers))
s.start()
# server(frontend_port)
s.join()
for w in processes:
w.join()
if __name__ == '__main__':
main()
This code works properly. But I want to use send_multipart() to send a tuple or a list that includes items with different types like [string, numpy_array, integer] but json can't handle numpy arrays. I am avoiding using pickle because I need it to be as fast as possible. I tried to convert the array to bytes too but it didn't work. (maybe I was doing it wrong I am not sure).
I appreciate if you can provide a working snippet of code.
Ideally, I want to do something like this:
socket.send_multipart([string, numpy_array, integer])
So I want to know what is the most efficient way of doing it.
I am using Python 3.6
msgpack and msgpack_numpy are the best option I could find.
Try this:
import time
import zmq
from zmq.devices.basedevice import ProcessDevice
from multiprocessing import Process
import numpy as np
import msgpack
import msgpack_numpy as m
def server(frontend_port, number_of_workers):
context = zmq.Context()
socket = context.socket(zmq.PUSH)
socket.connect("tcp://127.0.0.1:%d" % frontend_port)
for i in range(0,10):
arr = np.array([[[i,i],[i,i]],[[i,i],[i,i]]])
file_name = 'image file name or any other srting'
number = 10 # just an instance of an integer
msg = msgpack.packb((arr, number, file_name), default=m.encode, use_bin_type=True)
socket.send(msg, copy=False)
time.sleep(1)
for i in range(number_of_workers):
msg = msgpack.packb((b'STOP', b'STOP'), default=m.encode, use_bin_type=True)
socket.send(msg, copy=False)
return True
def worker(work_num, backend_port):
context = zmq.Context()
socket = context.socket(zmq.PULL)
socket.connect("tcp://127.0.0.1:%d" % backend_port)
while True:
task = socket.recv()
task = msgpack.unpackb(task, object_hook= m.decode, use_list=False, max_bin_len=50000000, raw=False)
if task[1] == b'STOP':
break
(arr, number, file_name) = task
print("Worker ",work_num, 'got message!', file_name)
return True
def main():
m.patch()
frontend_port = 3559
backend_port = 3560
number_of_workers = 2
streamerdevice = ProcessDevice(zmq.STREAMER, zmq.PULL, zmq.PUSH)
streamerdevice.bind_in("tcp://127.0.0.1:%d" % frontend_port )
streamerdevice.bind_out("tcp://127.0.0.1:%d" % backend_port)
streamerdevice.setsockopt_in(zmq.IDENTITY, b'PULL')
streamerdevice.setsockopt_out(zmq.IDENTITY, b'PUSH')
streamerdevice.start()
processes = []
for work_num in range(number_of_workers):
w = Process(target=worker, args=(work_num,backend_port))
processes.append(w)
w.start()
time.sleep(1)
s = Process(target=server, args=(frontend_port,number_of_workers))
s.start()
s.join()
for w in processes:
w.join()
if __name__ == '__main__':
main()
For my program, I have a file that writes random integers to a .CSV file.
from __future__ import absolute_import, division, print_function
from numpy.random import randint as randrange
import os, argparse, time
from tqdm import tqdm
def write_to_csv(filename, *args, newline = True):
write_string = ''
for arg in args:
if type(arg) == list:
for i in arg:
write_string += str(i) + ','
else:
write_string += str(arg) + ','
if newline:
write_string = write_string.rstrip(',') + '\n'
else:
write_string = write_string.rstrip(',')
with open(filename+'.csv', 'a') as file:
file.write(write_string)
def move_dir(dirname, parent = False):
if not parent:
dirname = str(dirname)
exists = os.path.isfile(dirname)
try:
os.mkdir(dirname)
os.chdir(dirname)
except FileExistsError:
os.chdir(dirname)
else:
os.chdir("..")
def calculate_probability(odds, exitmode = False, low_cpu = 0):
try:
file_count = 0
move_dir('Probability')
move_dir(str(odds))
d = {}
writelist = []
percentlist = []
for i in tqdm(range(odds)):
d[str(i)] = 0
writelist.append(f'Times {i}')
percentlist.append(f'Percent {i}')
while True:
if os.path.isfile(str(file_count)+'.csv'):
file_count += 1
else:
break
filename = str(file_count)
write_to_csv(filename, 'Number', 'Value')
rep = 500 * odds
if rep > 10000:
rep = 10000
for i in tqdm(range(rep)):
ran = randrange(odds)
ran = int(ran)
d[str(ran)] += 1
if i == 999:
write_to_csv(filename, i, ran+1, newline = False)
else:
write_to_csv(filename, i, ran+1)
if low_cpu:
time.sleep(0.01*float(low_cpu))
writelist2 = []
percentlist2 = []
for i in tqdm(range(odds)):
val = d[str(i)]
writelist2.append(val)
percentlist2.append(round(((val/rep)*100), 2))
if os.path.isfile('runs.csv'):
write_to_csv('runs', file_count, writelist2, percentlist2)
else:
write_to_csv('runs', 'Run #', writelist, percentlist)
write_to_csv('runs', file_count, writelist2, percentlist2)
if exitmode:
exit()
except(KeyboardInterrupt, SystemExit):
if exitmode:
os.remove(str(file_count)+'.csv')
exit()
else:
try:
os.system('cls')
print('User/program interrupted, lauching shutdown mode...')
os.remove(str(file_count)+'.csv')
print('Finilizaing current trial...')
os.chdir("..")
os.chdir("..")
except FileNotFoundError:
exit()
calculate_probability(odds, exitmode = True)
I also have a repetition system to do this multiple times.
def run_tests(times, odds, low_cpu = 0, shutdown = False):
for i in tqdm(range(times)):
calculate_probability(odds, low_cpu = low_cpu)
os.chdir("..")
os.chdir("..")
if shutdown:
os.system('shutdown /S /F /T 0 /hybrid')
However, if I were to run like 30 trails, it would take forever. So I decided to use the multiprocessing module to speed up the process. Because each run needs to write to the same file at the end, I had to collect the data and write them after the processes ended.
def calculate_probability(odds, low_cpu = 0):
try:
file_count = 0
move_dir('Probability')
move_dir(str(odds))
d = {}
writelist = []
percentlist = []
for i in tqdm(range(odds)):
d[str(i)] = 0
writelist.append(f'Times {i}')
percentlist.append(f'Percent {i}')
while True:
if os.path.isfile(str(file_count)+'.csv'):
file_count += 1
else:
break
filename = str(file_count)
write_to_csv(filename, 'Number', 'Value')
rep = 500 * odds
if rep > 10000:
rep = 10000
for i in range(rep):
ran = randrange(odds)
ran = int(ran)
d[str(ran)] += 1
if i == 999:
write_to_csv(filename, i, ran+1, newline = False)
else:
write_to_csv(filename, i, ran+1)
if low_cpu:
time.sleep(0.01*float(low_cpu))
writelist2 = []
percentlist2 = []
for i in range(odds):
val = d[str(i)]
writelist2.append(val)
percentlist2.append(round(((val/rep)*100), 2))
return (writelist, percentlist, writelist2, percentlist2)
except(KeyboardInterrupt, SystemExit):
try:
os.remove(str(file_count)+'.csv')
finally:
exit()
def worker(odds, returndict, num, low_cpu = 0):
returndict[f'write{num}'] = calculate_probability(odds, low_cpu = low_cpu)
os.chdir("..")
os.chdir("..")
os.system('cls')
def run_tests(times, odds, low_cpu = 0, shutdown = False):
print('Starting...')
manager = Manager()
return_dict = manager.dict()
job_list = []
for i in range(times):
p = Process(target=worker, args=(odds,return_dict,i), kwargs = {'low_cpu' : low_cpu})
job_list.append(p)
p.start()
try:
for proc in job_list:
proc.join()
except KeyboardInterrupt:
print('User quit program...')
time.sleep(5)
for proc in job_list:
proc.join()
exit()
else:
move_dir('Probability')
move_dir(str(odds))
if not os.path.isfile('runs.csv'):
write_to_csv('runs', return_dict.values()[0][0], return_dict.values()[0][1])
for value in return_dict.values():
write_to_csv('runs', value[2], value[3])
print('Done!')
finally:
if shutdown:
os.system('shutdown /S /F /T 0 /hybrid')
However, when I run this new code, there is one progressbar, and each process overwrites the bar, so the bar is flashing with random numbers, making the bar useful. I want to have a stack of bars, one for each process, that each update without interrupting the others. The bars do not need to be ordered; I just need to have an idea of how fast each process is doing their tasks.
STDOUT is just a stream, and all of your processes are attached to the same one, so there's no direct way to tell it to print the output from different processes on different lines.
Probably the simplest way to achieve this would be to have a separate process that is responsible for aggregating the status of all the other processes and reporting the results. You can use a multiprocessing.Queue to pass data from the worker threads to the status thread, then the status thread can print the status to stdout. If you want a stack of progress bars, you'll have to get a little creative with the formatting (essentially update all the progress bars at the same time and print them in the same order so they appear to stack up).
So Im trying to code a really simple Internet Download Manager Spoof with Python 2.7
It is supposed to query a files HTTP header, get the byte range and spread the download among a no.of threads(I hard-coded 2 for simplicity) according to the byte range and later join the file parts together again.
The problem is my console log tells me that only 1 thread is started.
[EDIT] The problem has been solved. Find the working code below.
Here is my source:
from __future__ import print_function
import threading
import urllib
import urllib2
import time
threads = []
# url to open
url = "http://www.sample-videos.com/video/mp4/720/big_buck_bunny_720p_1mb.mp4"
u = urllib.urlopen(url)
# define file
file_name = "test.mp4"
f = open(file_name, 'wb')
# open url and get header info
def get_file_size(url):
stream_size = u.info()['Content-Length']
end = stream_size
return end
start = 0
#get stream size
end = get_file_size(url)
# specify block size
block_sz = 512
#algo to divide work among 2 threads
def calculate_no_of_bytes_for_thread1():
full_stream_size = end
first_thread = {'start':0, 'end':(int(full_stream_size)/2)}
print(first_thread)
return first_thread
#algo to divide work among 2 threads
def calculate_no_of_bytes_for_thread2():
full_stream_size = end
second_thread= {'start':int(full_stream_size)/2,'end': int(full_stream_size)}
print(second_thread)
return second_thread
# download function
def download_thread(url ,id,start,end):
current_size = int(float(start)/1024)
total_size = int(float(end)/1024)
print ("Start at_"+str(current_size) + "Ends at_" + str(total_size))
# specify request range and init stream
req = urllib2.Request(url)
req.headers['Range'] = 'bytes=%s-%s' % (start, end)
data = urllib2.urlopen(req)
while True:
buffer = u.read(block_sz)
if not buffer:
break
start += len(buffer)
f.write(buffer)
thread_id = id
#percentage = (current_size * 100 / total_size)
status = str(thread_id) + "_" + str(current_size) + "_" +str(total_size)
print (status)
#starts 2 threads
def start_threads():
for i in range(2):
#if first loop, start thread 1
if(i==1):
start = calculate_no_of_bytes_for_thread1().get('start')
end = calculate_no_of_bytes_for_thread1().get('end')
print("Thread 1 started")
t = threading.Thread(target=download_thread, args=(url,i,start,end))
t.start()
threads.append( t)
#if second loop, start thread 1
if(i==2):
start = calculate_no_of_bytes_for_thread2().get('start')
end = calculate_no_of_bytes_for_thread2().get('end')
print("Thread 2 started")
t = threading.Thread(target=download_thread, args=(url,i,start,end))
t.start()
threads.append( t)
# Join threads back (order doesn't matter, you just want them all)
for i in threads:
i.join()
#start benchmarking
start_time = time.clock()
start_threads()
print ("Finito!")
end_time = time.clock()
benchmark = str(end_time - start_time)
print ("Download took_" +benchmark)
f.close()
And the output:
{'start': 0, 'end': 527868}
{'start': 0, 'end': 527868}
Thread 1 started
Start at_0Ends at_515
1_0_515
1_0_515
Finito!
Download took_6.97844422658
Working code:
from __future__ import print_function
import threading
import urllib
import urllib2
import time
threads = []
parts = {}
# url to open
url = "http://www.sample-videos.com/audio/mp3/india-national-anthem.mp3"
u = urllib.urlopen(url)
# define file
file_name = "test.mp3"
f = open(file_name, 'wb')
# open url and get header info
def get_file_size(url):
stream_size = u.info()['Content-Length']
file_size = stream_size
return file_size
start = 0
#get stream size
end = get_file_size(url)
# specify block size
block_sz = 512
#algo to divide work among 2 threads
def calculate_no_of_bytes_for_thread1():
full_stream_size = end
first_thread = {'start':0, 'end':(int(full_stream_size)/2)}
print(first_thread)
return first_thread
#algo to divide work among 2 threads
def calculate_no_of_bytes_for_thread2():
full_stream_size = end
second_thread= {'start':int(full_stream_size)/2,'end': int(full_stream_size)}
print(second_thread)
return second_thread
# download function
def download_thread(url ,id,start,end):
current_size = int(float(start)/1024)
total_size = int(float(end)/1024)
print ("Start at_"+str(current_size) + "Ends at_" + str(total_size))
# specify request range and init stream
req = urllib2.Request(url)
req.headers['Range'] = 'bytes=%s-%s' % (start, end)
while True:
buffer = u.read(block_sz)
if not buffer:
break
start += len(buffer)
f.write(buffer)
thread_id = id
status = "Thread ID_" +str(thread_id) + "Downloaded_" + str(int(start/1024)) + "Total_" +str(total_size)
print (status)
#starts 2 threads
def start_threads():
for i in range(2):
#if first loop, start thread 1
if(i==0):
start = calculate_no_of_bytes_for_thread1().get('start')
end = calculate_no_of_bytes_for_thread1().get('end')
print("Thread 1 started")
t = threading.Thread(target=download_thread, args=(url,i,start,end))
t.start()
threads.append( t)
#if second loop, start thread 2
if(i==1):
start = calculate_no_of_bytes_for_thread2().get('start')
end = calculate_no_of_bytes_for_thread2().get('end')
print("Thread 2 started")
t = threading.Thread(target=download_thread, args=(url,i,start,end))
t.start()
threads.append( t)
# Join threads back (order doesn't matter, you just want them all)
for i in threads:
i.join()
# Sort parts and you're done
# result = ''
# for i in range(2):
# result += parts[i*block_sz]
#start benchmarking
start_time = time.clock()
start_threads()
print ("Finito!")
end_time = time.clock()
benchmark = str(end_time - start_time)
print ("Download took_" +benchmark)
f.close()
You have:
for i in range(2):
if(i==1):
...
if(i==2):
...
But range(2) iterates over [0,1] not [1,2].
Save some trouble and just remove those 3 lines. The code to start the two threads can just run serially.
I have a small program to ping multiple IPs at the same time via sending 10 pings. This both records the results within a dict and also prints the status to a page.
However I want to allow the program to ping constantly and for the user to stop it rather then rely on a max ping count.
import os
import re
import time
import sys
import subprocess
import Queue
import threading
class pinger:
def __init__(self,hosts):
self.q = Queue.Queue()
self.all_results = []
self.hosts = hosts
def send_ping(self,q,ip):
self.q.put(self.record_results(ip))
def record_results(self,ip):
ping_count = 0
host_results = {
"host" : ip,
"device" : None,
"sent_count" : 0,
"success_count": 0,
"fail_count": 0,
"failed_perc": 0,
"curr_status": None
}
while ping_count < 10:
rc = subprocess.call(['ping', '-c', '1', '-W', '1', ip], stdout=open('/dev/null', 'w'), stderr=open('/dev/null', 'w'))
ping_count += 1
# update stats
host = host_results['host']
sent_count = host_results['sent_count']
success_count = host_results['success_count']
fail_count = host_results['fail_count']
failed_perc = host_results['failed_perc']
curr_status = host_results['curr_status']
sent_count += 1
if rc == 0:
success_count += 1
curr_status = "Successful Response"
else:
fail_count += 1
curr_status = "Request Timed Out"
failed_perc = ( fail_count / sent_count ) * 100
host_results.update({'failed_perc': failed_perc, 'fail_count': fail_count, 'success_count': success_count, 'curr_status': curr_status, 'sent_count': sent_count})
time.sleep(0.5)
print host_results
self.all_results.append(host_results)
return True
def go(self):
for i in self.hosts:
t = threading.Thread(target=self.send_ping, args = (self.q,i))
t.daemon = True
t.start()
Thanks,
You could change the while ping_count < 10 condition to while self.should_ping: (the variable would be initialized to True). Also, if there is a main loop where you wait to gather all the results, you can wrap it with try: except KeyboardInterrupt: and set pinger.should_ping to False in exception handler.
Otherwise, you could register to SIGINT signals, as mentioned by #bereal, and set the should_ping variable to False there.
I am new trying to implement either Parallel Python (PP) or async to multiprocess arcgis shapefile clipping. I have been successful with both pool_async and PP; however, it stalls forever on big files (and yes I tried making python access large addresses). Here is my code using PP, please offer any solutions and sorry for glaring errors if there are :-)
def ClipDo(F,M,O,OW = ""):
#for F in F:
print "\n"+"PID:%s"%(os.getpid())
arcpy.env.overwriteOutput = False
if OW == "":
pass
else:
arcpy.env.overwriteOutput = True
FPath = os.path.dirname(F)
F = os.path.basename(F)
ClipList = []
pattern = '*.shp'
for filename in M:
ClipList.append(filename)
clipN = str(os.path.splitext(os.path.basename(filename))[0])
if not os.path.isdir(O+"/"+clipN+"/"):
os.makedirs(O+"/"+clipN+"/")
#Counts files in clip directory
count = len(ClipList)
for num in range(0,count):
clip = ClipList[num]
clipN = str(os.path.splitext(os.path.basename(clip))[0])
OutShp = clipN +"_"+ F
try:
print "Clipping, Base File: %s Clip File: %s Output: %s" % (F,clip,O+"\\"+OutShp)
arcpy.Clip_analysis(os.path.join(FPath,F),os.path.join(M,clip), os.path.join(os.path.join(O+"\\",clipN),OutShp))
print "Clipping SUCCESS "
except:
print "Clipping FAILED " +F
def PP(F,M,O,OW):
print F
#~ # tuple of all parallel python servers to connect with
ncpus = 6
ncpus = ncpus
ppservers = ("localhost",)
#~ #ppservers = ("10.0.0.1",)
if len(sys.argv) > 1:
ncpus = int(sys.argv[1])
# Creates jobserver with ncpus workers
job_server = pp.Server(ncpus, ppservers=ppservers)
else:
#~ # Creates jobserver with automatically detected number of workers
job_server = pp.Server(ncpus,ppservers=ppservers)
print "Starting pp with", job_server.get_ncpus(), "workers"
jobs = []
start_time = time.time()
for f in F:
job = job_server.submit(ClipDo, (f,M,O,OW),(), ("arcpy","NullGeomFilter"))
jobs.append(job)
for job in jobs:
result = job()
print result
if result:
break
job_server.destroy()
print "\n"+"PID:%s"%(os.getpid())
print "Time elapsed: ", time.time() - start_time, "s"
Could it be that your big chunks are just too big for arcpy and that the parallelization is not the problem?
As a test, it might be good to run one of arg lists through your function with the big data interactively/locally to see if that's working at all. If it does, then you could move on to logging and debugging the parallel version.