I get an error when trying to run a command with joblib/multiprocessing in parallel:
Here the traceback:
Process PoolWorker-263:
Traceback (most recent call last):
File "/home/marcel/anaconda/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/home/marcel/anaconda/lib/python2.7/multiprocessing/process.py", line 114, in run
self._target(*self._args, **self._kwargs)
File "/home/marcel/anaconda/lib/python2.7/multiprocessing/pool.py", line 102, in worker
task = get()
File "/home/marcel/.local/lib/python2.7/site-packages/joblib/pool.py", line 363, in get
File "_objects.pyx", line 240, in h5py._objects.ObjectID.__cinit__ (h5py/_objects.c:2994)
TypeError: __cinit__() takes exactly 1 positional argument (0 given)
As you can see from the error message I work with data loaded using h5py. To complicate things further the routine I want to parallelize uses numba in one of its subroutines, but I hope that does not matter.
Here is a running example, which you can copy and paste:
from joblib import Parallel,delayed
import numpy as np
import h5py as h5
import os
def testfunc(h5data, row):
# some very boneheaded CPU work
data_slice = h5data[:,row,...]
ma = np.mean(data_slice, axis = 1)
x = row
return ma, x
def run():
data = np.random.random((100,100,100))
print data
f_out = h5.File('tmp.h5', 'w')
dset = f_out.create_dataset('mydata', data = data )
f_out.close()
f_in = h5.File('tmp.h5', 'r')
h5data = f_in['mydata']
pool = Parallel(n_jobs=-1,verbose=1,pre_dispatch='all')
results = pool(delayed(testfunc)(h5data, i) for i in range(h5data.shape[1]))
f_in.close()
os.remove('tmp.h5')
if __name__ == '__main__':
run()
Any ideas, what I'm doing wrong?
Edit: Okay at least I can exclude numba from the list of evildoers...
1You can try to replace Ë‹joblibwith [pathos][1] which replacespicklewithdill`. This solves generally all pickling issues.
Related
I'd like to generate a dynamically threads or processes in Python to consume each own queue.
My code: main.py
import cv2
import numpy as np
from classes import roi_process
import time
import os
import copy
import queue
import multiprocessing
roi_list = eval("[(0,0,639,720,1),(640,0,1280,720,2)]")
for _ROI in roi_list:
print("################# " + str(_ROI[4]))
vars()["FILA_"+str(_ROI[4])] = queue.Queue(maxsize=4)
vars()["T_"+str(_ROI[4])] = multiprocessing.Process(target = roi_process.RoiProcess, args = ( eval("FILA_"+str(_ROI[4])) , str(_ROI[4])), daemon=True)
for _ROI in roi_list:
eval("T_"+str(_ROI[4])).start()
classes/roi_process.py
import cv2
import queue
import numpy as np
import imutils
import time
import os
class RoiProcess:
def __init__(self, queue_pool = None, id_roi = 0):
self.id_roi = id_roi
self.queue_pool = queue_pool
print("Iniciou em thread o id: " + self.id_roi)
self.run()
def run(self):
i = 0
while True:
print(str(self.id_roi) + ": " + str(i))
i = i + 1
time.sleep(1)
This is generating the following error:
(tensorflow) C:\projects\car detector\semparar\AI_CARANDPLATE>python main.py
################# 1
################# 2
Traceback (most recent call last):
File "main.py", line 64, in <module>
eval("T_"+str(_ROI[4])).start()
File "C:\Users\MasterRoot\Anaconda3\envs\tensorflow\lib\multiprocessing\process.py", line 121, in start
self._popen = self._Popen(self)
File "C:\Users\MasterRoot\Anaconda3\envs\tensorflow\lib\multiprocessing\context.py", line 224, in _Popen
return _default_context.get_context().Process._Popen(process_obj)
File "C:\Users\MasterRoot\Anaconda3\envs\tensorflow\lib\multiprocessing\context.py", line 326, in _Popen
return Popen(process_obj)
File "C:\Users\MasterRoot\Anaconda3\envs\tensorflow\lib\multiprocessing\popen_spawn_win32.py", line 93, in __init__
reduction.dump(process_obj, to_child)
File "C:\Users\MasterRoot\Anaconda3\envs\tensorflow\lib\multiprocessing\reduction.py", line 60, in dump
ForkingPickler(file, protocol).dump(obj)
TypeError: cannot pickle '_thread.lock' object
(tensorflow) C:\projects\car detector\semparar\AI_CARANDPLATE>Traceback (most recent call last):
File "<string>", line 1, in <module>
File "C:\Users\MasterRoot\Anaconda3\envs\tensorflow\lib\multiprocessing\spawn.py", line 107, in spawn_main
new_handle = reduction.duplicate(pipe_handle,
File "C:\Users\MasterRoot\Anaconda3\envs\tensorflow\lib\multiprocessing\reduction.py", line 79, in duplicate
return _winapi.DuplicateHandle(
PermissionError: [WinError 5] Acesso negado
I really need to start dynamic threads or processes to consume every each poll that will be fed by a while True: in main.py
I will make a opencv frame reader and slice the main frame into many pieces.
After that I will feed a dynamic queue object with this information and each thread will process a predictor as I need.
I changed my code to:
FILA={}
T={}
#cria fila para cada ROI e instancia uma thread de obj para ler esta fila continuamente.
for _ROI in roi_list:
print("################# " + str(_ROI[4]))
FILA[_ROI[4]] = queue.Queue(maxsize=4)
T[_ROI[4]] = multiprocessing.Process(target = roi_process.RoiProcess, args = ( FILA[_ROI[4]] , str(_ROI[4])), daemon=True).start()
but its keepeing going to do the same error for threads.
Don't use standard Queue with multiprocessing, use:
from multiprocessing import Queue
When I attempt to load a sklearn logistic regression model from pickle using the following code
def _feed(gpu):
log_info("prediction process started")
model = pickle.load(open(model_path, 'rb'))
while True:
X = pending_q.get()
if not X:
log_info("stop pending thread")
return
# first_elem_non_neg_one_count = (X[-1][0] == -1).sum()
# log_info("non negative one inside X: %d" % first_elem_non_neg_one_count)
if X[0] > WINDOW_SIZE: # only begin prediction when we've got enough frames to compute means and stds
predicted = model.predict(X[-1])
# TODO: count consecutive and recent X # only
# sum_count = (predicted.flatten()[MOST_X_FRAME:] >= THRESHOLD).sum()
# log_info("***** got %d frames > %.1f in most recent %d frames" % (sum_count, THRESHOLD, abs(MOST_X_FRAME)))
log_info("frame: %d, predicted: %d" % (X[0], predicted))
predicted = X[:2] + predicted
else:
log_info("warming up, skip inference...")
predicted = X[:2] + [-1]
predicted_q.put(predicted) # TODO: batch prediction
I got the following error:
Exception in thread Thread-6:
Traceback (most recent call last):
File "/home/support/.pyenv/versions/3.6.6/lib/python3.6/threading.py", line 916, in _bootstrap_inner
self.run()
File "/home/support/.pyenv/versions/3.6.6/lib/python3.6/threading.py", line 864, in run
self._target(*self._args, **self._kwargs)
File "/app/xxxxx/04_predict_ps.py", line 106, in _feed
model = pickle.load(model_file)
File "/home/support/.pyenv/versions/va-worker-3.6.6/lib/python3.6/site-packages/sklearn/linear_model/__init__.py", line 12, in <module>
from .base import LinearRegression
File "/home/support/.pyenv/versions/va-worker-3.6.6/lib/python3.6/site-packages/sklearn/linear_model/base.py", line 38, in <module>
from ..preprocessing.data import normalize as f_normalize
File "/home/support/.pyenv/versions/va-worker-3.6.6/lib/python3.6/site-packages/sklearn/preprocessing/__init__.py", line 8, in <module>
from .data import Binarizer
File "/home/support/.pyenv/versions/va-worker-3.6.6/lib/python3.6/site-packages/sklearn/preprocessing/data.py", line 19, in <module>
from scipy import stats
File "/home/support/.pyenv/versions/va-worker-3.6.6/lib/python3.6/site-packages/scipy/stats/__init__.py", line 345, in <module>
from .stats import *
File "/home/support/.pyenv/versions/va-worker-3.6.6/lib/python3.6/site-packages/scipy/stats/stats.py", line 171, in <module>
from . import distributions
File "/home/support/.pyenv/versions/va-worker-3.6.6/lib/python3.6/site-packages/scipy/stats/distributions.py", line 13, in <module>
from . import _continuous_distns
File "/home/support/.pyenv/versions/va-worker-3.6.6/lib/python3.6/site-packages/scipy/stats/_continuous_distns.py", line 6692, in <module>
pairs = list(globals().items())
RuntimeError: dictionary changed size during iteration
I save the model using the below:
sk_logistic_regr = LogisticRegression()
sk_logistic_regr.fit(x_train, y_train)
with open(SK_MODEL_NAME, 'wb') as file:
pickle.dump(sk_logistic_regr, file)
I am not sure if it is because I am not closing the pickle file properly when I kill my script. Any idea on the reason causing this? It seems that the model loading code only succeeds for the first time.
P.S. the _feed function is being run on a separate thread dedicated for inferencing
Meanwhile as a workaround, I used from joblib import dump, load which is recommended in sklearn doc which fixes my problem.
I was running the following code to test the broadcasting files using mpi_bcast. Basically, I created a text file "test.txt" at the root and broadcast to the other nodes.
import json
import numpy as np
import multiprocessing
import time
import sys
import os
import math
from scipy.special import comb
import itertools
from mpi4py import MPI
comm = MPI.COMM_WORLD
size = comm.Get_size()
rank = comm.Get_rank()
if rank == 0:
with open('test.txt', 'w') as f:
for x in range(50):
f.write('{}\n'.format(x))
data = open('test.txt', 'r')
print(rank)
else:
data = None
data = comm.bcast(data, root=0)
if rank != 0:
f_out = open('test.txt', 'w')
for i, line in enumerate(data):
f_out.write(line)
f_out.close()
print('rank',rank)
Then when I run mpiexec -n 3 python3 test.py over 3 processes, I get teh following error messages.
Traceback (most recent call last):
File "test.py", line 26, in <module>
data = comm.bcast(data, root=0)
File "mpi4py/MPI/Comm.pyx", line 1257, in mpi4py.MPI.Comm.bcast
File "mpi4py/MPI/msgpickle.pxi", line 629, in mpi4py.MPI.PyMPI_bcast
File "mpi4py/MPI/msgpickle.pxi", line 104, in mpi4py.MPI.Pickle.dump
File "mpi4py/MPI/msgpickle.pxi", line 91, in mpi4py.MPI.Pickle.cdumps
TypeError: cannot serialize '_io.TextIOWrapper' object
I think the error happens at this line: data = comm.bcast(data, root=0).
Is it because I cannot pass a file object to the bcast function? How can I fix this error?
I have a two dimensional function and I want to compute the elements of the function on the grid points but the two loops over rows and columns are very slow and I want to use multiprocessing to increase the speed of the code. I have written the following code to do two loops:
from multiprocessing import Pool
#Grid points
ra = np.linspace(25.1446, 25.7329, 1000)
dec = np.linspace(-10.477, -9.889, 1000)
#The 2D function
def like2d(x,y):
stuff=[RaDec, beta, rho_c_over_sigma_c, zhalo, rho_crit]
m=3e14
c=7.455
param=[x, y, m, c]
return reduced_shear( param, stuff, observed_g, g_err)
pool = Pool(processes=12)
def data_stream(a, b):
for i, av in enumerate(a):
for j, bv in enumerate(b):
yield (i, j), (av, bv)
def myfunc(args):
return args[0], like2d(*args[1])
counter,likelihood = pool.map(myfunc, data_stream(ra, dec))
But I got the following error message:
Process PoolWorker-1:
Traceback (most recent call last):
File "/user/anaconda/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/user/anaconda/lib/python2.7/multiprocessing/process.py", line 114, in run
self._target(*self._args, **self._kwargs)
File "/user/anaconda/lib/python2.7/multiprocessing/pool.py", line 102, in worker
task = get()
File "/user/anaconda/lib/python2.7/multiprocessing/queues.py", line 376, in get
return recv()
AttributeError: 'module' object has no attribute 'myfunc'
Process PoolWorker-2:
Traceback (most recent call last):
File "/user/anaconda/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/user/anaconda/lib/python2.7/multiprocessing/process.py", line 114, in run
self._target(*self._args, **self._kwargs)
File "/user/anaconda/lib/python2.7/multiprocessing/pool.py", line 102, in worker
task = get()
File "/user/anaconda/lib/python2.7/multiprocessing/queues.py", line 376, in get
return recv()
AttributeError: 'module' object has no attribute 'myfunc'
Process PoolWorker-3:
Traceback (most recent call last):
File "/user/anaconda/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/user/anaconda/lib/python2.7/multiprocessing/process.py", line 114, in run
self._target(*self._args, **self._kwargs)
File "/user/anaconda/lib/python2.7/multiprocessing/pool.py", line 102, in worker
task = get()
File "/user/anaconda/lib/python2.7/multiprocessing/queues.py", line 376, in get
return recv()
AttributeError: 'module' object has no attribute 'myfunc'
Process PoolWorker-4:
Everything is defined and I do not understand why this error message raised!! Can anybody point out what might be wrong?
Another approach to do loops with multiprocessing and save the results in a 2d array:
#Grid points
ra = np.linspace(25.1446, 25.7329, 1000)
dec = np.linspace(-10.477, -9.889, 1000)
#The 2D function
def like2d(x,y):
stuff=[RaDec, beta, rho_c_over_sigma_c, zhalo, rho_crit]
m=3e14
c=7.455
param=[x, y, m, c]
return reduced_shear( param, stuff, observed_g, g_err)
shared_array_base = multiprocessing.Array(ctypes.c_double, ra.shape[0]*dec.shape[0])
shared_array = np.ctypeslib.as_array(shared_array_base.get_obj())
shared_array = shared_array.reshape( ra.shape[0],dec.shape[0])
# Parallel processing
def my_func(i, def_param=shared_array):
shared_array[i,:] = np.array([float(like2d(ra[j],dec[i])) for j in range(ra.shape[0])])
print "processing to estimate likelihood in 2D grids......!!!"
start = time.time()
pool = multiprocessing.Pool(processes=12)
pool.map(my_func, range(dec.shape[0]))
print shared_array
end = time.time()
print end - start
You have to create the Pool after the worker function (myfunc) definition. Creating the Pool causes Python to fork your worker processes right at that point, and the only things that will be defined in the children are the functions defined above the Pool definition. Also, map will return a list of tuples (one for each object yielded by data_stream), not a single tuple. So you need this:
from multiprocessing import Pool
#Grid points
ra = np.linspace(25.1446, 25.7329, 1000)
dec = np.linspace(-10.477, -9.889, 1000)
#The 2D function
def like2d(x,y):
stuff=[RaDec, beta, rho_c_over_sigma_c, zhalo, rho_crit]
m=3e14
c=7.455
param=[x, y, m, c]
return reduced_shear( param, stuff, observed_g, g_err)
def data_stream(a, b):
for i, av in enumerate(a):
for j, bv in enumerate(b):
yield (i, j), (av, bv)
def myfunc(args):
return args[0], like2d(*args[1])
if __name__ == "__main__":
pool = Pool(processes=12)
results = pool.map(myfunc, data_stream(ra, dec)) # results is a list of tuples.
for counter,likelihood in results:
print("counter: {}, likelihood: {}".format(counter, likelihood))
I added the if __name__ == "__main__": guard, which isn't necessary on POSIX platforms, but would be necessary on Windows (which doesn't support os.fork()).
I have a function ComparePatchMany written in numpy, which does some basic matrix functions (dot product, diagonal, etc), and due to the size of matrices I'm using, is too slow. In order to achieve some speedup, I want to run calls to this function in parallel. Because of memory issues, I can't seem to call this on any more than 100 stacked matrices at a time. So simply running ComparePatchMany on a giant matrix is out (though it works in MatLab).
What I have right now is:
def comparePatchManyRunner(tex_flat,imMask_flat,s_tex,metrics,i):
metrics[i] = ComparePatchMany.main(tex_flat[imMask_flat==1,:],np.reshape(s_tex[:,i],(-1,1)))
# N = 100
def main(TexLib,tex,OperationMask,N,gpu=0):
if gpu:
print 'ERROR: GPU Capability not set'
else:
tex_flat = np.array([tex.flatten('F')]).T
CreateGrid = np.ones((TexLib.Gr.l_y.shape[1],TexLib.Gr.l_x.shape[1]))
PatchMap = np.nan*CreateGrid
MetricMap = np.nan*CreateGrid
list_of_patches = np.argwhere(CreateGrid>0)
for i in range(list_of_patches.shape[0]):
y,x = list_of_patches[i]
imMask = TexLib.obtainMask(y,x)
Box = [TexLib.Gr.l_x[0,x],TexLib.Gr.l_x[-1,x],TexLib.Gr.l_y[0,y],TexLib.Gr.l_y[-1,y]]
imMaskO = imMask
imMask = imMask & OperationMask
imMask_flat = np.dstack((imMask,imMask,imMask))
if gpu:
print 'ERROR! GPU Capability not yet implemented'
# TODO
else:
imMask_flat = imMask_flat.flatten('F')
if np.sum(imMask)<8:
continue
indd_s = np.random.randint(TexLib.NumTexs,size=(1,N*5))
s_tex = TexLib.ImW[imMask_flat==1][:,np.squeeze(indd_s)]
s_tex = s_tex.astype('float32')
if gpu:
print 'ERROR! GPU Capability not yet implemented'
# TODO
else:
metrics = np.zeros((N*5,1))
shared_arr = multiprocessing.Array('d',metrics)
processes = [multiprocessing.Process(target=comparePatchManyRunner, args=(tex_flat,imMask_flat,s_tex,shared_arr,i)) for i in xrange(N*5)]
for p in processes:
p.start()
for p in processes:
p.join()
metrics = shared_arr
print metrics
I think this may be creating 500 processes, which could be an issue. One problem I keep running into with this and previous versions is IOError: [Errno 32] Broken pipe, which originates from p.start().
I'm working on Windows with Python 2.7, NumPy 1.8, and SciPy 0.13.2.
EDIT:
Comments suggested I use pools. So I'm trying this:
metrics = np.zeros((N*5,1))
shared_arr = multiprocessing.Array('d',metrics,lock=False)
po = multiprocessing.Pool(processes=2)
po.map_async(comparePatchManyRunner,((tex_flat,imMask_flat,s_tex,shared_arr,idex) for idex in xrange(N*5)))
But it doesn't seem to be writing anything to shared_arr, and I keep getting a PicklingError:
Exception in thread Thread-29:
Traceback (most recent call last):
File "C:\Python27\lib\threading.py", line 810, in __bootstrap_inner
self.run()
File "C:\Python27\lib\threading.py", line 763, in run
self.__target(*self.__args, **self.__kwargs)
File "C:\Python27\lib\multiprocessing\pool.py", line 342, in _handle_tasks
put(task)
PicklingError: Can't pickle <class 'multiprocessing.sharedctypes.c_double_Array_500'>: attribute lookup multiprocessing.sharedctypes.c_double_Array_500 failed