Related
I have some difficulties with choosing a random row(point in my case) from my np array. I want to do that with probabilities for each point( so I have a P_i np array in which each row is the probability for a point). I tried to do it with np.random.choice and get "it's must be a 1-D array" so I did np.random.choice on the number of the rows so I get a random index of row. But how do I do it with a probability for each point?
You can use np.choice with a probability distribution that sums up to 1.
Getting probabilities that sum up to 1
Reshaping
If your probablities already sum up to 1, then you simply want to squeeze your probability vector:
# Example of probability vector
probs = np.array([[0.1, 0.2, 0.5, 0.2]])
# array([[0.1, 0.2, 0.5, 0.2]])
probs.shape
# > (1, 4)
p_squeezed = probs.squeeze()
# > array([0.1, 0.2, 0.5, 0.2])
p_squeezed.shape
# > (4,)
Getting a proper probability distribution
If your own probs don't add up to 1, then you can apply a division by the sum or a softmax.
Just generating random data:
import numpy as np
# Random 2D points
points = np.random.randint(0,10, size=(10,2))
# random independant probabilities
probs = np.random.rand(10).reshape(-1, 1)
data = np.hstack((probs, points))
print(data)
# > array([[0.01402932, 5. , 5. ],
# [0.01454579, 5. , 6. ],
# [0.43927214, 1. , 7. ],
# [0.36369286, 3. , 7. ],
# [0.09703463, 9. , 9. ],
# [0.56977406, 1. , 4. ],
# [0.0453545 , 4. , 2. ],
# [0.70413767, 4. , 4. ],
# [0.72133774, 7. , 1. ],
# [0.27297051, 3. , 6. ]])
Applying softmax:
from scipy.special import softmax
scale_softmax = softmax(data[:,0])
# > array([0.07077797, 0.07081454, 0.1082876 , 0.10040494, 0.07690364,
# 0.12338291, 0.0730302 , 0.14112644, 0.14357482, 0.09169694])
Applying division by the sum:
scale_divsum = data[: ,0] / data[:, 0].sum()
# > array([0.00432717, 0.00448646, 0.13548795, 0.11217647, 0.02992911,
# 0.17573962, 0.01398902, 0.21718238, 0.22248752, 0.08419431])
Here are the cumulative distributions of the scaling functions I proposed :
Softmax makes it more similarly likely to pick any point than division by the sum, but the latter probably better fits your needs.
Picking a random row
Now you can use np.random.choice and give it your probability distribution to the parameter p:
rand_idx = np.random.choice(np.arange(len(data)), p=scale_softmax)
data[rand_idx]
# > array([0.70413767, 4. , 4. ])
# or just the point:
data[rand_idx, 1:]
# > array([4., 4.])
I have an image and its mask picked from a competition hosted in kaggle. The shape of the image is (512,512,3) and the mask is (512,512,1). After applying a function(flipping) on an image, the shape remains the same. However, before applying the operation when I try to access the mask such as (print mask[:,:,0]), I get a matrix,
[[0. 0. 0. ... 0. 0. 0.]
[0. 0. 0. ... 0. 0. 0.]
[0. 0. 0. ... 0. 0. 0.]
...
[0. 0. 0. ... 0. 0. 0.]
[0. 0. 0. ... 0. 0. 0.]
[0. 0. 0. ... 0. 0. 0.]]
but after applying the operation, and try to access the mask (print mask[:,:,0]), I get the following error
Traceback (most recent call last):
File "Augmentation.py", line 94, in <module>
plot_img_and_mask_transformed(img,mask,img_flip,mask_flip)
File "Augmentation.py", line 36, in plot_img_and_mask_transformed
print(mask_tr[:,:,0])
IndexError: too many indices for array
The function I applied was
def random_flip(img,mask,u=0.5):
if np.random.random() < u :
img = cv.flip(img,0)
mask = cv.flip(mask,0)
return img, mask
img, mask = get_image_and_mask(img_id)
img_tr,mask_tr = random_flip(img,mask)
plot(img,mask,img_tr,mask_tr)
The shape of the image and the mask before flipping
((512, 512, 3), (512, 512, 1))
The shape of the image and the mask after flipping
((512, 512, 3), (512, 512))
Can someone help me out what's happening behind the scenes?
CODE
def get_image_and_mask(img_id):
img = image.load_img(join(data_dir,'train','%s.jpg' % img_id),target_size=(input_size,input_size))
img = image.img_to_array(img)
mask = image.load_img(join(data_dir,'train_masks','%s_mask.gif' % img_id), grayscale=True,target_size=(input_size,input_size))
mask = image.img_to_array(mask)
img,mask = img / 255., mask/ 255.
return img, mask
def plot_img_and_mask(img,mask):
fig, axs = plt.subplots(ncols=2, figsize=(10,5),sharex=True,sharey=True)
axs[0].imshow(img)
axs[1].imshow(mask[:,:,0])
for ax in axs:
ax.set_xlim(0,input_size)
ax.axis('off')
fig.tight_layout()
plt.show()
def plot_img_and_mask_transformed(img, mask, img_tr, mask_tr):
fig, axs=plt.subplots(ncols=4,figsize=(16,4),sharex=True,sharey=True)
axs[0].imshow(img)
axs[1].imshow(mask[:,:,0])
print(mask[:,:,0])
print(mask_tr[:,:,0])
axs[2].imshow(img_tr)
axs[3].imshow(mask_tr)
for ax in axs:
ax.set_xlim(0,input_size)
ax.axis('off')
fig.tight_layout()
plt.show()
def random_flip(img,mask,u=0.5):
# Why do we have to check less than u
if np.random.random() < u :
img = cv.flip(img,0)
mask = cv.flip(mask,0)
return img, mask
def rotate(x,theta,row_axis=0,col_axis=1,channel_axis=2,fill_mode='nearest',cval=0):
rotation_matrix = np.array([
[np.cos(theta),-np.sin(theta),0],
[np.sin(theta),np.cos(theta),0],
[0,0,1]
])
h, w = x.shape[row_axis], x.shape[col_axis]
transform_matrix = image.transform_matrix_offset_center(rotation_matrix,h,w)
x = image.apply_transform(x,transform_matrix,channel_axis,fill_mode,cval)
return x
def random_rotate(img, mask, rotate_limit=(-20,20), u=0.5):
if np.random.random() < u:
theta = np.pi/ 180 * np.random.uniform(rotate_limit[0], rotate_limit[1])
img = rotate(img,theta)
mask = rotate(mask,theta)
return img, mask
if __name__== '__main__':
input_size = 512
data_dir = '../data/carvana-image-masking-challenge'
np.random.seed(1987)
df_train = pd.read_csv(join(data_dir,'train_masks.csv'),usecols=['img'])
df_train['img_id']=df_train['img'].map(lambda s:s.split('.')[0])
df_train.head(3)
img_ids=df_train['img_id'].values
np.random.shuffle(img_ids)
img_id=img_ids[0]
img,mask=get_image_and_mask(img_id)
print((img.shape,mask.shape))
plot_img_and_mask(img,mask)
img_flip,mask_flip = random_flip(img,mask,u=1)
print((img_flip.shape,mask_flip.shape))
plot_img_and_mask_transformed(img,mask,img_flip,mask_flip)
OUTPUT
Using TensorFlow backend.
C:\Users\JamesJohnson\AppData\Local\Programs\Python\Python35\lib\site- packages\keras_preprocessing\image.py:492: UserWarning: grayscale is deprecated. Please use color_mode = "grayscale"
warnings.warn('grayscale is deprecated. Please use '
> ((512, 512, 3), (512, 512, 1))
> ((512, 512, 3), (512, 512))
[[0. 0. 0. ... 0. 0. 0.]
[0. 0. 0. ... 0. 0. 0.]
[0. 0. 0. ... 0. 0. 0.]
...
[0. 0. 0. ... 0. 0. 0.]
[0. 0. 0. ... 0. 0. 0.]
[0. 0. 0. ... 0. 0. 0.]]
Traceback (most recent call last):
File "Augmentation.py", line 94, in <module>
plot_img_and_mask_transformed(img,mask,img_flip,mask_flip)
File "Augmentation.py", line 36, in plot_img_and_mask_transformed
print(mask_tr[:,:,0])
IndexError: too many indices for array
It looks like OpenCV dumps the singleton dimension when you flip the mask. You'll need to reintroduce it after you flip.
mask_flip = mask_flip[..., None]
A more convenient way is to modify your method so the mask is returned with the singleton dimension after you flip in case you lose it. This way you don't have to do this every time you flip and the method takes care of that instead.
def random_flip(img,mask,u=0.5):
# Why do we have to check less than u
if np.random.random() < u:
img = cv.flip(img,0)
mask = cv.flip(mask,0)
if len(mask.shape) == 2:
mask = mask[..., None]
return img, mask
BTW as a minor note, you have a comment that asks why you have to check for less than u in the method. Remember that the np.random.random method generates a value between 0 and 1 uniformly. Suppose you chose u = 0.3. This means that there is a 30% chance that you will choose a value between between 0 and 0.3 and a 70% chance that you will choose a value between 0.3 and 1. Loosely, this means that if u = 0.3, there is a 30% chance that the if condition is run and you thus flip the image and mask. Therefore, the u controls the probability that a flip of the image and mask will happen.
Is there a way to utilize the array indices within a vectorized numpy equation?
Specifically, I have this looping code that sets each value of a 2d array to the distance to some arbitrary center point.
img=np.ndarray((size[0],size[1]))
for x in range(size[0]):
for y in range(size[1]):
img[x,y]=math.sqrt((x-center[0])**2+(y-center[1])**2)
How might I vectorize that?
You can solve this easily using broadcasting:
import numpy as np
size = (64, 64)
center = (32, 32)
x = np.arange(size[0])
y = np.arange(size[1])
img = np.sqrt((x - center[0]) ** 2 + (y[:, None] - center[1]) ** 2)
Some help from Pandas would make this task relatively easy:
import itertools
import pandas as pd
import numpy as np
# get all of the xy pairs
xys = pd.DataFrame(list(itertools.product(range(size[0]), range(size[1]))))
# calculate distance
xys["distance"] = np.sqrt((xys[0] - center[0]) ** 2 + (xys[1] - center[1]) ** 2)
# transform to a 2d array
img = xys.set_index([0, 1])["distance"].unstack()
# if you want just the Numpy array, not a Pandas DataFrame
img.values
Yes, there is.
import numpy as np
size = (6, 4)
center = (3, 2)
img_xy = np.array([[(x, y) for x in range(size[0])] for y in range(size[1])])
img = np.sum((img_xy - center) ** 2, axis=2) ** 0.5
print('\nPlan1:\n', img)
img = np.linalg.norm(img_xy - center, axis=2)
print('\nPlan2:\n', img)
You will get this:
Plan1:
[[3.60555128 2.82842712 2.23606798 2. 2.23606798 2.82842712]
[3.16227766 2.23606798 1.41421356 1. 1.41421356 2.23606798]
[3. 2. 1. 0. 1. 2. ]
[3.16227766 2.23606798 1.41421356 1. 1.41421356 2.23606798]]
Plan2:
[[3.60555128 2.82842712 2.23606798 2. 2.23606798 2.82842712]
[3.16227766 2.23606798 1.41421356 1. 1.41421356 2.23606798]
[3. 2. 1. 0. 1. 2. ]
[3.16227766 2.23606798 1.41421356 1. 1.41421356 2.23606798]]
If you have any question, you could ask me.
Given a 3D numpy array of shape (256, 256, 256), how would I make a solid sphere shape inside? The code below generates a series of increasing and decreasing circles but is diamond shaped when viewed in the two other dimensions.
def make_sphere(arr, x_pos, y_pos, z_pos, radius=10, size=256, plot=False):
val = 255
for r in range(radius):
y, x = np.ogrid[-x_pos:n-x_pos, -y_pos:size-y_pos]
mask = x*x + y*y <= r*r
top_half = arr[z_pos+r]
top_half[mask] = val #+ np.random.randint(val)
arr[z_pos+r] = top_half
for r in range(radius, 0, -1):
y, x = np.ogrid[-x_pos:size-x_pos, -y_pos:size-y_pos]
mask = x*x + y*y <= r*r
bottom_half = arr[z_pos+r]
bottom_half[mask] = val#+ np.random.randint(val)
arr[z_pos+2*radius-r] = bottom_half
if plot:
for i in range(2*radius):
if arr[z_pos+i].max() != 0:
print(z_pos+i)
plt.imshow(arr[z_pos+i])
plt.show()
return arr
EDIT: pymrt.geometry has been removed in favor of raster_geometry.
DISCLAIMER: I am the author of both pymrt and raster_geometry.
If you just need to have the sphere, you can use the pip-installable module raster_geometry, and particularly raster_geometry.sphere(), e.g:
import raster_geometry as rg
arr = rg.sphere(3, 1)
print(arr.astype(np.int_))
# [[[0 0 0]
# [0 1 0]
# [0 0 0]]
# [[0 1 0]
# [1 1 1]
# [0 1 0]]
# [[0 0 0]
# [0 1 0]
# [0 0 0]]]
internally, this is implemented as an n-dimensional superellipsoid generator, you can check its source code for details.
Briefly, the (simplified) code would reads like this:
import numpy as np
def sphere(shape, radius, position):
"""Generate an n-dimensional spherical mask."""
# assume shape and position have the same length and contain ints
# the units are pixels / voxels (px for short)
# radius is a int or float in px
assert len(position) == len(shape)
n = len(shape)
semisizes = (radius,) * len(shape)
# genereate the grid for the support points
# centered at the position indicated by position
grid = [slice(-x0, dim - x0) for x0, dim in zip(position, shape)]
position = np.ogrid[grid]
# calculate the distance of all points from `position` center
# scaled by the radius
arr = np.zeros(shape, dtype=float)
for x_i, semisize in zip(position, semisizes):
# this can be generalized for exponent != 2
# in which case `(x_i / semisize)`
# would become `np.abs(x_i / semisize)`
arr += (x_i / semisize) ** 2
# the inner part of the sphere will have distance below or equal to 1
return arr <= 1.0
and testing it:
# this will save a sphere in a boolean array
# the shape of the containing array is: (256, 256, 256)
# the position of the center is: (127, 127, 127)
# if you want is 0 and 1 just use .astype(int)
# for plotting it is likely that you want that
arr = sphere((256, 256, 256), 10, (127, 127, 127))
# just for fun you can check that the volume is matching what expected
# (the two numbers do not match exactly because of the discretization error)
print(np.sum(arr))
# 4169
print(4 / 3 * np.pi * 10 ** 3)
# 4188.790204786391
I am failing to get how your code exactly works, but to check that this is actually producing spheres (using your numbers) you could try:
arr = sphere((256, 256, 256), 10, (127, 127, 127))
# plot in 3D
import matplotlib.pyplot as plt
from skimage import measure
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1, projection='3d')
verts, faces, normals, values = measure.marching_cubes(arr, 0.5)
ax.plot_trisurf(
verts[:, 0], verts[:, 1], faces, verts[:, 2], cmap='Spectral',
antialiased=False, linewidth=0.0)
plt.show()
Other approaches
One could implement essentially the same with a combination of np.linalg.norm() and np.indices():
import numpy as np
def sphere_idx(shape, radius, position):
"""Generate an n-dimensional spherical mask."""
assert len(position) == len(shape)
n = len(shape)
position = np.array(position).reshape((-1,) + (1,) * n)
arr = np.linalg.norm(np.indices(shape) - position, axis=0)
return arr <= radius
producing the same results (sphere_ogrid is sphere from above):
import matplotlib.pyplot as plt
funcs = sphere_ogrid, sphere_idx
fig, axs = plt.subplots(1, len(funcs), squeeze=False, figsize=(4 * len(funcs), 4))
d = 500
n = 2
shape = (d,) * n
position = (d // 2,) * n
size = (d // 8)
base = sphere_ogrid(shape, size, position)
for i, func in enumerate(funcs):
arr = func(shape, size, position)
axs[0, i].imshow(arr)
However, this is going to be substantially slower and requires much more temporary memory n_dim * shape of the output.
The benchmarks below seems to support the speed assessment:
base = sphere_ogrid(shape, size, position)
for func in funcs:
print(f"{func.__name__:20s}", np.allclose(base, arr), end=" ")
%timeit -o func(shape, size, position)
# sphere_ogrid True 1000 loops, best of 5: 866 µs per loop
# sphere_idx True 100 loops, best of 5: 4.15 ms per loop
size = 100
radius = 10
x0, y0, z0 = (50, 50, 50)
x, y, z = np.mgrid[0:size:1, 0:size:1, 0:size:1]
r = np.sqrt((x - x0)**2 + (y - y0)**2 + (z - z0)**2)
r[r > radius] = 0
Nice question. My answer to a similar question would be applicable here also.
You can try the following code. In the below mentioned code AA is the matrix that you want.
import numpy as np
from copy import deepcopy
''' size : size of original 3D numpy matrix A.
radius : radius of circle inside A which will be filled with ones.
'''
size, radius = 5, 2
''' A : numpy.ndarray of shape size*size*size. '''
A = np.zeros((size,size, size))
''' AA : copy of A (you don't want the original copy of A to be overwritten.) '''
AA = deepcopy(A)
''' (x0, y0, z0) : coordinates of center of circle inside A. '''
x0, y0, z0 = int(np.floor(A.shape[0]/2)), \
int(np.floor(A.shape[1]/2)), int(np.floor(A.shape[2]/2))
for x in range(x0-radius, x0+radius+1):
for y in range(y0-radius, y0+radius+1):
for z in range(z0-radius, z0+radius+1):
''' deb: measures how far a coordinate in A is far from the center.
deb>=0: inside the sphere.
deb<0: outside the sphere.'''
deb = radius - abs(x0-x) - abs(y0-y) - abs(z0-z)
if (deb)>=0: AA[x,y,z] = 1
Following is an example of the output for size=5 and radius=2 (a sphere of radius 2 pixels inside a numpy array of shape 5*5*5):
[[[0. 0. 0. 0. 0.]
[0. 0. 0. 0. 0.]
[0. 0. 1. 0. 0.]
[0. 0. 0. 0. 0.]
[0. 0. 0. 0. 0.]]
[[0. 0. 0. 0. 0.]
[0. 0. 1. 0. 0.]
[0. 1. 1. 1. 0.]
[0. 0. 1. 0. 0.]
[0. 0. 0. 0. 0.]]
[[0. 0. 1. 0. 0.]
[0. 1. 1. 1. 0.]
[1. 1. 1. 1. 1.]
[0. 1. 1. 1. 0.]
[0. 0. 1. 0. 0.]]
[[0. 0. 0. 0. 0.]
[0. 0. 1. 0. 0.]
[0. 1. 1. 1. 0.]
[0. 0. 1. 0. 0.]
[0. 0. 0. 0. 0.]]
[[0. 0. 0. 0. 0.]
[0. 0. 0. 0. 0.]
[0. 0. 1. 0. 0.]
[0. 0. 0. 0. 0.]
[0. 0. 0. 0. 0.]]]
I haven't printed the output for the size and radius that you had asked for (size=32 and radius=4), as the output will be very long.
Here is how to create voxels space without numpy, the main idea that you calculate distance between center and voxel and if voxel in radius you will create.
from math import sqrt
def distance_dimension(xyz0 = [], xyz1 = []):
delta_OX = pow(xyz0[0] - xyz1[0], 2)
delta_OY = pow(xyz0[1] - xyz1[1], 2)
delta_OZ = pow(xyz0[2] - xyz1[2], 2)
return sqrt(delta_OX+delta_OY+delta_OZ)
def voxels_figure(figure = 'sphere', position = [0,0,0], size = 1):
xmin, xmax = position[0]-size, position[0]+size
ymin, ymax = position[1]-size, position[1]+size
zmin, zmax = position[2]-size, position[2]+size
voxels = []
if figure == 'cube':
for local_z, world_z in zip(range(zmax-zmin), range(zmin, zmax)):
for local_y, world_y in zip(range(ymax-ymin), range(ymin, ymax)):
for local_x, world_x in zip(range(xmax-xmin), range(xmin, xmax)):
voxels.append([world_x,world_y,world_z])
elif figure == 'sphere':
for local_z, world_z in zip(range(zmax-zmin), range(zmin, zmax)):
for local_y, world_y in zip(range(ymax-ymin), range(ymin, ymax)):
for local_x, world_x in zip(range(xmax-xmin), range(xmin, xmax)):
radius = distance_dimension(xyz0 = [world_x, world_y,world_z], xyz1 = position)
if radius < size:
voxels.append([world_x,world_y,world_z])
return voxels
voxels = voxels_figure(figure = 'sphere', position = [0,0,0], size = 3)
After you will get voxels indexes, you can apply ~ones for cube matrix.
Instead of using loops, I propose to use a meshgrid + sphere equation + np.where
import numpy as np
def generate_sphere(volumeSize):
x_ = np.linspace(0,volumeSize, volumeSize)
y_ = np.linspace(0,volumeSize, volumeSize)
z_ = np.linspace(0,volumeSize, volumeSize)
r = int(volumeSize/2) # radius can be changed by changing r value
center = int(volumeSize/2) # center can be changed here
u,v,w = np.meshgrid(x_, y_, z_, indexing='ij')
a = np.power(u-center, 2)+np.power(v-center, 2)+np.power(w-center, 2)
b = np.where(a<=r*r,1,0)
return b
I am a beginner at python and numpy and I need to compute the matrix logarithm for each "pixel" (i.e. x,y position) of a matrix-valued image of dimension NxMx3x3. 3x3 is the dimensions of the matrix at each pixel.
The function I have written so far is the following:
def logm_img(im):
from scipy import linalg
dimx = im.shape[0]
dimy = im.shape[1]
res = zeros_like(im)
for x in range(dimx):
for y in range(dimy):
res[x, y, :, :] = linalg.logm(asmatrix(im[x,y,:,:]))
return res
Is it ok?
Is there a way to avoid the two nested loops ?
Numpy can do that. Just call numpy.log:
>>> import numpy
>>> a = numpy.array(range(100)).reshape(10, 10)
>>> b = numpy.log(a)
__main__:1: RuntimeWarning: divide by zero encountered in log
>>> b
array([[ -inf, 0. , 0.69314718, 1.09861229, 1.38629436,
1.60943791, 1.79175947, 1.94591015, 2.07944154, 2.19722458],
[ 2.30258509, 2.39789527, 2.48490665, 2.56494936, 2.63905733,
2.7080502 , 2.77258872, 2.83321334, 2.89037176, 2.94443898],
[ 2.99573227, 3.04452244, 3.09104245, 3.13549422, 3.17805383,
3.21887582, 3.25809654, 3.29583687, 3.33220451, 3.36729583],
[ 3.40119738, 3.4339872 , 3.4657359 , 3.49650756, 3.52636052,
3.55534806, 3.58351894, 3.61091791, 3.63758616, 3.66356165],
[ 3.68887945, 3.71357207, 3.73766962, 3.76120012, 3.78418963,
3.80666249, 3.8286414 , 3.8501476 , 3.87120101, 3.8918203 ],
[ 3.91202301, 3.93182563, 3.95124372, 3.97029191, 3.98898405,
4.00733319, 4.02535169, 4.04305127, 4.06044301, 4.07753744],
[ 4.09434456, 4.11087386, 4.12713439, 4.14313473, 4.15888308,
4.17438727, 4.18965474, 4.20469262, 4.21950771, 4.2341065 ],
[ 4.24849524, 4.26267988, 4.27666612, 4.29045944, 4.30406509,
4.31748811, 4.33073334, 4.34380542, 4.35670883, 4.36944785],
[ 4.38202663, 4.39444915, 4.40671925, 4.41884061, 4.4308168 ,
4.44265126, 4.4543473 , 4.46590812, 4.47733681, 4.48863637],
[ 4.49980967, 4.51085951, 4.52178858, 4.53259949, 4.54329478,
4.55387689, 4.56434819, 4.57471098, 4.58496748, 4.59511985]])