Develop Reference

How to convert mat file->numpy file->nifti file to match with nnU-Net data format?

I'm currently documenting how to convert each data type to be compatible with a new deeplearning framework. I'll cut out redundant code in the futrue :)
The following code can be executed on VScode interactive window.
The code has two part.
convert mat file to npy file (.mat -> .npy)
convert npy file to nifti file (.npy -> nii.gz) and add specific name in the path to match with nnU-Net data format. See the nnU-Net dataset_conversion.md if you're interested in it.
How it actually works?
1)10000001.mat -> 10000001.npy
2)10000001.npy -> AORTA_001_0000.nii.gz
The path can be adjusted by a individual user.
#%%
import numpy as np
import nibabel as nb
import pathlib
import numpy as np
from torch.utils.data import Dataset
import scipy.io
root_data = '/mnt/intern/code/dataset/test/original'
root_label = '/mnt/intern/code/dataset/test/label'
examples = []
examples2 = []
data_files = list(pathlib.Path(root_data).iterdir())
label_files = list(pathlib.Path(root_label).iterdir())
for fname in sorted(data_files):
examples += [fname]
for fname2 in sorted(label_files):
examples2 += [fname2]
for i in range(len(data_files)):
fname = examples[i]
fname2 = examples2[i]
data_name = str(pathlib.Path(fname))
label_name = str(pathlib.Path(fname2))
# d = np.load(data_name); l = np.load(label_name)
d_load = scipy.io.loadmat(data_name);
l_load = scipy.io.loadmat(label_name) # matfile data load
data = d_load['data'];
label = l_load['label'] # (512, 512, 251)
np.save('/mnt/intern/mat2npy/original/' + str(fname).split('.')[0][-8:], data)
np.save('/mnt/intern/mat2npy/label/' + str(fname).split('.')[0][-8:], label)
#%%
# Name change to match with nnU-Net data format
import numpy as np
import nibabel as nb
import pathlib
import numpy as np
from torch.utils.data import Dataset
import scipy.io
import numpy as np
import nibabel as nib
root_data = '/mnt/intern/mat2npy/imagesTr'
root_label = '/mnt/intern/mat2npy/labelsTr'
examples = []
examples2 = []
data_files = list(pathlib.Path(root_data).iterdir())
label_files = list(pathlib.Path(root_label).iterdir())
for fname in sorted(data_files):
examples += [fname]
for fname2 in sorted(label_files):
examples2 += [fname2]
for i in range(len(data_files)):
fname = examples[i]
fname2 = examples2[i]
data_name = str(pathlib.Path(fname))
label_name = str(pathlib.Path(fname2))
# d = np.load(data_name); l = np.load(label_name)
d_load = np.load(data_name);
l_load = np.load(label_name) # matfile data load
data = d_load
label = l_load # (512, 512, 251)
data = np.array(data, dtype=np.float32) # You need to replace normal array by yours
label = np.array(label, dtype=np.float32)
affine = np.eye(4)
nifti_data = nib.Nifti1Image(data, affine)
nifti_label = nib.Nifti1Image(label, affine)
nib.save(nifti_data, '/mnt/intern/mat2npy/imagesTr/' + 'AORTA_' + str(fname).split('.')[0][-3:] + '_0000.nii.gz') # Here you put the path + the extionsion 'nii' or 'nii.gz'
nib.save(nifti_label, '/mnt/intern/mat2npy/labelsTr/' + 'AORTA_' + str(fname).split('.')[0][-3:] + '_0000.nii.gz')

How to use tfp.density.Mixture with JointDistributionCoroutine

I'm trying to define a model function for MCMC.
The idea is to have a mixture of two distributions controlled with a probability ratio.
One of my attempts would look like this:
import tensorflow as tf
import tensorflow_probability as tfp
tfd = tfp.distributions
root = tfd.JointDistributionCoroutine.Root
def model_fn():
rv_p = yield root(tfd.Sample(tfd.Uniform(0.0,1.0),1))
catprobs = tf.stack([rv_p, 1.-rv_p],0)
rv_cat = tfd.Categorical(probs=catprobs)
rv_norm1 = tfd.Sample(tfd.Normal(0.0,1.0),1)
rv_norm2 = tfd.Sample(tfd.Normal(3.0,1.0),1)
rv_mix = yield tfd.Mixture(cat=rv_cat,
components=[
rv_norm1,
rv_norm2,
])
jd = tfd.JointDistributionCoroutine(model_fn)
jd.sample(2)
The code fails with:
ValueError: components[0] batch shape must be compatible with cat shape and other component batch shapes ((2, 2) vs ())
Could you give me an example of how to use Mixture distribution in a way that allows "any" shape of inputs?
I'm using tensorflow 2.4.1 and tensorflow_probability 0.12.1 with python 3.6

I figured it out. For reference here is a sample code:
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
import tensorflow as tf
import tensorflow_probability as tfp
import matplotlib.pyplot as plt
tfd = tfp.distributions
tfb = tfp.bijectors
import numpy as np
from time import time
numdata = 10000
data = np.random.normal(0.0,1.0,numdata).astype(np.float32)
data[int(numdata/2):] = 0.0
_=plt.hist(data,30,density=True)
root = tfd.JointDistributionCoroutine.Root
def dist_fn(rv_p,rv_mu):
rv_cat = tfd.Categorical(probs=tf.stack([rv_p, 1.-rv_p],-1))
rv_norm = tfd.Normal(rv_mu,1.0)
rv_zero = tfd.Deterministic(tf.zeros_like(rv_mu))
rv_mix = tfd.Independent(
tfd.Mixture(cat=rv_cat,
components=[rv_norm,rv_zero]),
reinterpreted_batch_ndims=1)
return rv_mix
def model_fn():
rv_p = yield root(tfd.Sample(tfd.Uniform(0.0,1.0),1))
rv_mu = yield root(tfd.Sample(tfd.Uniform(-1.,1. ),1))
rv_mix = yield dist_fn(rv_p,rv_mu)
jd = tfd.JointDistributionCoroutine(model_fn)
unnormalized_posterior_log_prob = lambda *args: jd.log_prob(args + (data,))
n_chains = 1
p_init = [0.3]
p_init = tf.cast(p_init,dtype=tf.float32)
mu_init = 0.1
mu_init = tf.stack([mu_init]*n_chains,axis=0)
initial_chain_state = [
p_init,
mu_init,
]
bijectors = [
tfb.Sigmoid(), # p
tfb.Identity(), # mu
]
step_size = 0.01
num_results = 50000
num_burnin_steps = 50000
kernel=tfp.mcmc.TransformedTransitionKernel(
inner_kernel=tfp.mcmc.HamiltonianMonteCarlo(
target_log_prob_fn=unnormalized_posterior_log_prob,
num_leapfrog_steps=2,
step_size=step_size,
state_gradients_are_stopped=True),
bijector=bijectors)
kernel = tfp.mcmc.SimpleStepSizeAdaptation(
inner_kernel=kernel, num_adaptation_steps=int(num_burnin_steps * 0.8))
#XLA optim
#tf.function(autograph=False, experimental_compile=True)
def graph_sample_chain(*args, **kwargs):
return tfp.mcmc.sample_chain(*args, **kwargs)
st = time()
trace,stats = graph_sample_chain(
num_results=num_results,
num_burnin_steps=num_burnin_steps,
current_state=initial_chain_state,
kernel=kernel)
et = time()
print(et-st)
ptrace, mutrace = trace
plt.subplot(121)
_=plt.hist(ptrace.numpy(),100,density=True)
plt.subplot(122)
_=plt.hist(mutrace.numpy(),100,density=True)
print(np.mean(ptrace),np.mean(mutrace))

Python SDF reader failure

I want to run this python script. I installed the SDF reader in linux by the following command in my home directory
python -m pip install --upgrade sdf
and it seems to be installed successfully.
The python script is the following
import sdf
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt
import numpy as np
import os
from numpy import ma
from matplotlib import colors, ticker, cm
from matplotlib.mlab import bivariate_normal
from matplotlib.colors import ListedColormap
if __name__ == "__main__":
print ('This is main of module "test2d.py"')
######## Constant defined here ########
pi = 3.1415926535897932384626
q0 = 1.602176565e-19 # C
m0 = 9.10938291e-31 # kg
v0 = 2.99792458e8 # m/s^2
kb = 1.3806488e-23 # J/K
mu0 = 4.0e-7*pi # N/A^2
epsilon0 = 8.8541878176203899e-12 # F/m
h_planck = 6.62606957e-34 # J s
wavelength= 1.0e-6
frequency = v0*2*pi/wavelength
exunit = m0*v0*frequency/q0
bxunit = m0*frequency/q0
denunit = frequency**2*epsilon0*m0/q0**2
print 'electric field unit: '+str(exunit)
print 'magnetic field unit: '+str(bxunit)
print 'density unit nc: '+str(denunit)
font = {'family' : 'helvetica',
'color' : 'black',
'weight' : 'normal',
'size' : 20,
}
n=47
data = sdf.read("./"+str(n).zfill(4)+".sdf",dict=True)
header=data['Header']
time=header['time']
x = data['Grid/Grid_mid'].data[0]/1.0e-6
y = data['Grid/Grid_mid'].data[1]/1.0e-6
y = y[600:1799]
X, Y = np.meshgrid(x, y)
It gives me following error:
Traceback (most recent call last):
File "epochvis.py", line 45, in <module>
data = sdf.read("./"+str(n).zfill(4)+".sdf",dict=True)
AttributeError: 'module' object has no attribute 'read'
Any ideas? Thank you in advance.

sdf does not have a read function.
try typing the following in your python shell
import sdf
help(sdf)
you will see
FUNCTIONS
load(filename, objectname='/', unit=None, scale_units=None)
Load a dataset or group from an SDF file
save(filename, group)
Save an SDF group to a file
validate(obj)
Validate an sdf.Group or sdf.Dataset

RectilinearGridSource from tvtk.RectilinearGrid()

I am trying to construct tvtk.RectilinearGridSource from tvtk.RectilinearGrid object in order to add it to the mayavi.engine.
I used to do this:
import mayavi.mlab as mlab
r = tvtk.RectilinearGrid()
r.point_data.scalars = data.ravel()
r.point_data.scalars.name = 'Temperature'
d = mlab.pipline.add_dataset(r)
but instead I would prefer to call it this way:
from mayavi.api import Engine
e = Engine()
e.start()
s = e.new_scene()
src = tvtk.RectilinearGridSource()
and then link src with r i.e., with my RectilinearGrid defined before.
Is there any way to do this ?

I've found an answer:
r = tvtk.RectilinearGrid()
r.point_data.scalars = data.ravel()
r.point_data.scalars.name = 'Temperature'
from mayavi.sources.vtk_data_source import VTKDataSource
src = VTKDataSource(data=r)
e.add_source(d)

rpy2 dtw missing argument window.size

I'm using the R DTW package with rpy2. I would like to be able specify a window type and size for running the DTW analysis.
I have run the following code:
import numpy as np
import rpy2.robjects as robjects
import rpy2.robjects.numpy2ri
rpy2.robjects.numpy2ri.activate()
r = robjects.r
r('library("dtw")')
query = np.array([0.0,1.0,2.0,3.0])
reference = np.array([0.0,1.9,2.4,3.0])
# Attempt 1:
kwargs = {'step':r("asymmetric"),'window_type':r("sakoeChibaWindow"),'window_size':r("as.integer(\"3\")")}
alig = r.dtw(query, reference, **kwargs)
# Attempt 2:
alig = r.dtw(query, reference, keep=r('TRUE'), step=r('asymmetric'),window_type=r('sakoeChibaWindow'),window_size="as.integer(\"3\")")
# Attempt 3:
alig = r.dtw(query, reference, keep=r('TRUE'), step=r('asymmetric'),window_type=r('sakoeChibaWindow'),window_size=3)
# Note: The line of code below works correctly.
# alig = r.dtw(query, reference, keep=r('TRUE'), step=r('asymmetric'))
robjects.globalenv["alignment"] = alig
print r('alignment$distance')
I get the following error message:
Error in abs(jw - iw) <= window.size : 'window.size' is missing
Traceback (most recent call last):
File "testrdtw.py", line 19, in <module>
alig = r.dtw(query, reference, **kwargs)
File "/Users/jsmith/Dropbox/IW/env/lib/python2.7/site-packages/rpy2/robjects/functions.py", line 86, in __call__
return super(SignatureTranslatedFunction, self).__call__(*args, **kwargs)
File "/Users/jsmith/Dropbox/IW/env/lib/python2.7/site-packages/rpy2/robjects/functions.py", line 35, in __call__
res = super(Function, self).__call__(*new_args, **new_kwargs)
rpy2.rinterface.RRuntimeError: Error in abs(jw - iw) <= window.size : 'window.size' is missing
How do I properly specify the window.size argument such that it is passed correctly?
I'm quite new to R and rpy so I could very well be using these libraries incorrectly.
Any suggestions,hints, or help greatly appreciated.
-- js

Use importr():
from rpy2.robjects.packages import importr
dtw = importr('dtw')
alig = dtw.dtw(query, reference, keep=True,
step='asymmetric',
window_type='sakoeChibaWindow',
window_size=3)

This works for me:
import numpy as np
import rpy2.robjects.numpy2ri
from rpy2.robjects.packages import importr
rpy2.robjects.numpy2ri.activate()
R = rpy2.robjects.r
DTW = importr('dtw')
x = np.array([0.0, 1.0, 2.0, 3.0])
y = np.array([0.0, 1.9, 2.4, 3.0])
alignment1 = R.dtw(x, y, keep=True, dist_method="Euclidean",step_pattern=DTW.asymmetric,type="sakoechiba")
alignment2 = R.dtw(x, y, keep=True, dist_method="Euclidean",step_pattern=DTW.symmetric1,type="itakura")
alignment3 = R.dtw(x, y, keep=True, dist_method="Euclidean", step_pattern=DTW.symmetric2, type=DTW.sakoeChibaWindow, window_size=2)
dist1 = alignment1.rx('distance')[0][0]
dist2 = alignment2.rx('distance')[0][0]
dist3= alignment3.rx('distance')[0][0]
print(dist1)
#1.0
print(dist2)
#1.3
print(dist3)
#1.3
The documentation states:"window.type can also be an user-defined windowing function. See dtwWindowingFunctions for all available windowing functions"
There u can fix the window.size.
Hope it helps