from __future__ import division
import scipy.io
import numpy as np
import math
from math import sin
from math import cos
mat = np.zeros((1024,1024,360 ),dtype=np.float32)
x = scipy.io.loadmat('/home/prakriti/Project/A.mat')
A = np.array((x.values()))
mat[:,:,0:120] = A[0][:,:,:]
del x
del A
y = scipy.io.loadmat('/home/prakriti/Project/B.mat')
B = np.array((y.values()))
mat[:,:,120:240] = B[0][:,:,:]
del y
del B
z = scipy.io.loadmat('/home/prakriti/Project/C.mat')
C = np.array((z.values()))
mat[:,:,240:360] = C[0][:,:,:]
del z
del C
s = np.linspace(0,1023,1024)
v = np.linspace(0,1023,1024)
ss,vv = np.meshgrid(s,v)
zz = ss**2 + vv**2
print zz
I have been getting the following error for this. Can anyone explain me what is the problem here? I am trying to make a 3-D matrix mat with data that is available to me. I have 49GB of RAM available. Why do I still get memory error?
Traceback (most recent call last):
File "/home/prakriti/Project/fdk_new.py", line 11, in <module>
x = scipy.io.loadmat('/home/prakriti/Project/A.mat')
File "/usr/lib/python2.7/dist-packages/scipy/io/matlab/mio.py", line 152, in loadmat
matfile_dict = MR.get_variables(variable_names)
File "/usr/lib/python2.7/dist-packages/scipy/io/matlab/mio5.py", line 270, in get_variables
hdr, next_position = self.read_var_header()
File "/usr/lib/python2.7/dist-packages/scipy/io/matlab/mio5.py", line 223, in read_var_header
stream = BytesIO(dcor.decompress(data))
MemoryError
Providing an answer because a comment would not format correctly.
Try not creating extra memory. The GIL takes a bit of time to release it.
Instead of :
x = scipy.io.loadmat('/home/prakriti/Project/A.mat')
A = np.array((x.values()))
mat[:,:,0:120] = A[0][:,:,:]
del x
del A
do:
mat[:,:,0:120] = np.array((scipy.io.loadmat('/home/prakriti/Project/A.mat').values()))[0][:,:,:]
etc...
Otherwise, we need more info. Like the size of the mat files.
Related
I want to create a linear operator in python to solve Ax = b where A is a large-scale dense Matrix of float64. Since matrix A cause both performance and memory problems I thought about creating a customized operator as follows:
from numpy import ones
from numpy.linalg import inv
import scipy.sparse.linalg
from sklearn.datasets import make_spd_matrix
n = 100
def solver(A, b):
return inv(A).dot(b)
M = make_spd_matrix(n, random_state=11)
print(M.shape)
solverFunc = scipy.sparse.linalg.LinearOperator((n, n), matvec=solver)
solverFunc.matvec(M, ones((n, 1)))
However, I get the following error:
Traceback (most recent call last):
File "C:\Users\anoir\Desktop\CG_accelerator\inversion\main.py", line 15, in <module>
solverFunc = LinearOperator((n, n), matvec=solver)
File "C:\ProgramData\Anaconda3\envs\inversion\lib\site-packages\scipy\sparse\linalg\interface.py", line 521, in __init__
self._init_dtype()
File "C:\ProgramData\Anaconda3\envs\inversion\lib\site-packages\scipy\sparse\linalg\interface.py", line 178, in _init_dtype
self.dtype = np.asarray(self.matvec(v)).dtype
File "C:\ProgramData\Anaconda3\envs\inversion\lib\site-packages\scipy\sparse\linalg\interface.py", line 232, in matvec
y = self._matvec(x)
File "C:\ProgramData\Anaconda3\envs\inversion\lib\site-packages\scipy\sparse\linalg\interface.py", line 530, in _matvec
return self.__matvec_impl(x)
TypeError: solver() missing 1 required positional argument: 'b'
What seems to be the problem here? I followed the documentation but there is nothing about custom LinearOperator.
The linear operator only takes one parameter. You can get around this by using a closure as shown below:
from numpy.linalg import inv
import numpy as np
import scipy.sparse.linalg
from scipy.sparse import random
import timeit
n = 100
def solver_closure(A):
# This is the outer enclosing function
def solver(b):
return inv(A).dot(b)
return solver # returns the nested function
M = np.random.rand(n, n)
b = range(n)
print(M.shape)
solverFunc = scipy.sparse.linalg.LinearOperator((n, n), matvec=solver_closure(M))
def test100():
x = solverFunc.matvec(b)
print(np.matmul(M,x))
print(timeit.timeit("test100()", setup="from __main__ import test100",number=10))
When running the code below, I'm getting a TypeError that says:
"File "_vq.pyx", line 342, in scipy.cluster._vq.update_cluster_means
TypeError: type other than float or double not supported"
from PIL import Image
import scipy, scipy.misc, scipy.cluster
NUM_CLUSTERS = 5
im = Image.open('d:/temp/test.jpg')
ar = scipy.misc.fromimage(im)
shape = ar.shape
ar = ar.reshape(scipy.product(shape[:2]), shape[2])
codes, dist = scipy.cluster.vq.kmeans(ar, NUM_CLUSTERS)
vecs, dist = scipy.cluster.vq.vq(ar, codes)
counts, bins = scipy.histogram(vecs, len(codes))
peak = codes[scipy.argmax(counts)]
print 'Most frequent color: %s (#%s)' % (peak, ''.join(chr(c) for c in peak).encode('hex'))
I have no idea how to fix this.
Update:
Full traceback:
Traceback (most recent call last):
File "...\temp.py", line 110, in <module>
codes, dist = scipy.cluster.vq.kmeans2(ar, NUM_CLUSTERS)
File "...\site-packages\scipy\cluster\vq.py", line 642, in kmeans2
new_code_book, has_members = _vq.update_cluster_means(data, label, nc)
File "_vq.pyx", line 342, in scipy.cluster._vq.update_cluster_means
TypeError: type other than float or double not supported
Doing:
ar = ar.reshape(scipy.product(shape[:2]), shape[2])
print(ar.dtype)
you will see, that you call kmeans with data of type uint8.
As kmeans, in theory, is defined on a d-dimensional real vector, scipy also does not like it (as given in the error)!
So just do:
ar = ar.reshape(scipy.product(shape[:2]), shape[2]).astype(float)
Casting like that is making my example run until the print, which also needs to be changed to reflect the given types.
Traceback (most recent call last):
File "test.py", line 37, in <module>
print convLayer1.output.shape.eval({x:xTrain})
File "/Volumes/TONY/anaconda/lib/python2.7/site-packages/theano/gof/graph.py", line 415, in eval
rval = self._fn_cache[inputs](*args)
File "/Volumes/TONY/anaconda/lib/python2.7/site-packages/theano/compile/function_module.py", line 513, in __call__
allow_downcast=s.allow_downcast)
File "/Volumes/TONY/anaconda/lib/python2.7/site-packages/theano/tensor/type.py", line 180, in filter
"object dtype", data.dtype)
TypeError
And here is my code:
import scipy.io as sio
import numpy as np
import theano.tensor as T
from theano import shared
from convnet3d import ConvLayer, NormLayer, PoolLayer, RectLayer
from mlp import LogRegr, HiddenLayer, DropoutLayer
from activations import relu, tanh, sigmoid, softplus
dataReadyForCNN = sio.loadmat("DataReadyForCNN.mat")
xTrain = dataReadyForCNN["xTrain"]
# xTrain = np.random.rand(10, 1, 5, 6, 2).astype('float64')
xTrain.shape
dtensor5 = T.TensorType('float64', (False,)*5)
x = dtensor5('x') # the input data
yCond = T.ivector()
# input = (nImages, nChannel(nFeatureMaps), nDim1, nDim2, nDim3)
kernel_shape = (5,6,2)
fMRI_shape = (51, 61, 23)
n_in_maps = 1 # channel
n_out_maps = 5 # num of feature maps, aka the depth of the neurons
num_pic = 2592
layer1_input = x
# layer1_input.eval({x:xTrain}).shape
# layer1_input.shape.eval({x:numpy.zeros((2592, 1, 51, 61, 23))})
convLayer1 = ConvLayer(layer1_input, n_in_maps, n_out_maps, kernel_shape, fMRI_shape,
num_pic, tanh)
print convLayer1.output.shape.eval({x:xTrain})
It is really weird as the error was not thrown in Jupyter (but it takes long long time to run and finally the kernel is down I really don't know why), but as I move it to the shell and run python fileName.py the error was thrown.
The problem lies in loadmat from scipy. The typeerror you are getting is thrown by this code in Theano:
if not data.flags.aligned:
...
raise TypeError(...)
Now, when you create a new array in numpy from raw data, it would usually be aligned:
>>> a = np.array(2)
>>> a.flags.aligned
True
But if you savemat / loadmat it, the value of the flag gets lost:
>>> savemat('test', {'a':a})
>>> a2 = loadmat('test')['a']
>>> a2.flags.aligned
False
(seems like this particular issue is discussed here)
One quick and dirty way to address it is to create a new numpy array from the array you loaded:
>>> a2 = loadmat('test')['a']
>>> a3 = np.array(a2)
>>> a3.flags.aligned
True
So, for your code:
dataReadyForCNN = np.array(sio.loadmat("DataReadyForCNN.mat"))
I am trying to use scikit-cuda's wrappers for the cuSOLVER functions, in particular I want to execute cusolverDnSgesvd to compute full-matrix single precision SVD on a matrix of real numbers.
Using the code here and here as a reference, I managed to get this far:
import pycuda.autoinit
import pycuda.driver as drv
import pycuda.gpuarray as gpuarray
import numpy as np
from skcuda import cusolver
handle = cusolver.cusolverDnCreate()
m = 50
n = 25
a = np.asarray(np.random.random((m, n)))
a_gpu = gpuarray.to_gpu(a)
ldu = m
ldvt = n
s_gpu = gpuarray.empty(min(m, n), np.float32)
u_gpu = gpuarray.empty((ldu, m), np.float32)
vh_gpu = gpuarray.empty((n, n), np.float32)
work_size = cusolver.cusolverDnSgesvd_bufferSize(handle, m, n)
work = gpuarray.empty((m,n), np.float32)
u_gpu, s_gpu, vh_gpu = cusolver.cusolverDnSgesvd(
handle=handle,
jobu='A',
jobvt='A',
m=m,
n=n,
A=a,
lda=m,
S=s_gpu,
U=u_gpu,
ldu=ldu,
VT=vh_gpu,
ldvt=ldvt,
Work=work,
Lwork=work_size,
rwork=None,
devInfo=0
)
But the code isn't working, probably because I'm messing up with types.
Traceback (most recent call last):
File "/home/vektor/PycharmProjects/yancut/test_svd.py", line 44, in <module>
devInfo=0
File "/home/vektor/anaconda3/lib/python3.4/site-packages/skcuda/cusolver.py", line 577, in cusolverDnSgesvd
int(A), lda, int(S), int(U),
TypeError: only length-1 arrays can be converted to Python scalars
How should I provide all the arguments so that the SVD is executed in a proper way?
UPDATE1:
After using this question as reference, I edited my code and I'm getting a new error.
import pycuda.autoinit
import pycuda.driver as drv
import pycuda.gpuarray as gpuarray
import numpy as np
import ctypes
from skcuda import cusolver
rows = 20
cols = 10
a = np.asarray(np.random.random((rows, cols)))
a_gpu = gpuarray.to_gpu(a.copy())
lda = rows
u_gpu = gpuarray.empty((rows, rows), np.float32)
v_gpu = gpuarray.empty((cols, cols), np.float32)
s_gpu = gpuarray.empty(cols, np.float32)
devInfo = gpuarray.zeros(1, np.int32)
handle = cusolver.cusolverDnCreate()
worksize = cusolver.cusolverDnSgesvd_bufferSize(handle, rows, cols)
print("SIZE", worksize)
Workspace = gpuarray.empty(worksize, np.float32)
svd_status = cusolver.cusolverDnSgesvd(
handle=handle,
jobu='A',
jobvt='A',
m=rows,
n=cols,
A=a_gpu.ptr,
lda=rows,
S=s_gpu.ptr,
U=u_gpu.ptr,
ldu=rows,
VT=v_gpu.ptr,
ldvt=cols,
Work=Workspace.ptr,
Lwork=worksize,
rwork=Workspace.ptr,
devInfo=devInfo.ptr
)
status = cusolver.cusolverDnDestroy(handle)
And I'm getting a new error
Traceback (most recent call last):
File "/home/vektor/PycharmProjects/yancut/test_svd.py", line 53, in <module>
devInfo=devInfo.ptr
File "/home/vektor/anaconda3/lib/python3.4/site-packages/skcuda/cusolver.py", line 579, in cusolverDnSgesvd
Lwork, int(rwork), int(devInfo))
ctypes.ArgumentError: argument 2: <class 'TypeError'>: wrong type
It now seems that I'm doing something wrong with devInfo
From the documentation it looks like each of the matrices (so A, S, U, VT) need to be passed as device pointers. So for PyCUDA gpuarrays, pass A.ptr rather than A. etc and it should work.
I'm trying to initialize 2D surface in PyCUDA and fill it with values from NumPy 2D array.The idea, as I get it, is
open drv.ArrayDescriptor,
create drv.Array using this descriptor,
copy data from NumPy array with drv.Memcpy2D,
do set_array for SurfaceReference.
But still I have pycuda._driver.LogicError at the last step. A minimal example of what I'm doing:
import numpy as np
import pycuda.driver as drv
import pycuda.autoinit
from pycuda.compiler import SourceModule
mod = SourceModule("surface<void, cudaSurfaceType2D> fld_srf;")
def numpy2d_to_array(np_array):
h, w = np_array.shape
descr = drv.ArrayDescriptor()
descr.width = w
descr.height = h
descr.format = drv.dtype_to_array_format(np_array.dtype)
descr.num_channels = 1
descr.flags = 0
device_array = drv.Array(descr)
copy = drv.Memcpy2D()
copy.set_src_host(np_array)
copy.set_dst_array(device_array)
copy.width_in_bytes = copy.src_pitch = np_array.strides[0]
copy.src_height = copy.height = h
copy(aligned=True)
return device_array
fld = np.random.random_integers(-30, 30, (1920, 1080)).astype(np.int32)
srf = mod.get_surfref('fld_srf')
srf_arr = numpy2d_to_array(fld.copy())
srf.set_array(srf_arr)
The code above throws following exception:
Traceback (most recent call last):
File "./testsurface.py", line 30, in <module>
srf.set_array(srf_arr)
pycuda._driver.LogicError: cuSurfRefSetArray failed: invalid value
Any ideas how to do this correctly? Or at least why this error appears?
It might have something to do with the flags: in the 3D case, you have to set
descr.flags = drv.array3d_flags.SURFACE_LDST
to allow binding to the surface according to
this.
I don't find the 2D equivalent for pycuda though.