I am attempting to pull out these y values from neural network. The current problem seems to be numpy not multiplying the matrix as I expected. I have included the code and output for your review. Thank you in advance for any insights you can provide.
def columnToRow(column):
newarray = np.array([column])
return newarray
def calcIndividualOutput(indivInputs,weights,biases):
# finds the resulting y values for one set of input data
I_transposed= columnToRow(indivInputs)
output = np.multiply(I_transposed, weights) + biases
return output
def getOutputs(inputs,weights,biases):
# iterates over each set of inputs to find corresponding outputs
# returns output matrix
i_len = len(inputs)-1
outputs = []
for i in range(0,i_len):
result = calcIndividualOutput(inputs[i],weights,biases)
outputs.append(np.tanh(result))
if (i==i_len):
print("Final Input reached:", i)
return outputs
# Test Single line of Outputs should
#print("Resulting Outputs0:\n\n",resultingOutputs[0,0:])
# Testing
currI=data[0]
Itrans=columnToRow(currI)
print(" THE CURRENT I0\n\n",currI,"\n\n")
print("transposed I:\n\n",Itrans,"\n\n")
print("Itrans shape:\n\n",Itrans.shape,"\n\n")
print("Current biases:\n\n",model_l1_b,"\n\n")
print("Current biases shape:\n\n",model_l1_b.shape,"\n\n")
print("B trans:",b_trans,"\n\n")
print("B trans shape:",b_trans.shape,"\n\n")
print("Current weights:\n\n",model_l1_W,"\n\n")
print("Transposed weights\n\n",w_transposed,"\n\n")
print("wtrans shape:\n\n",w_transposed.shape,"\n\n")
#Test calcIndividualOutput
testOutput= calcIndividualOutput(currI,w_transposed,b_trans)
print("Test calcIndividualOutput:\n\n",testOutput,"\n\n")
print("Test calcIndividualOutput Shape:\n\n",testOutput.shape,"\n\n")
# Transpose weights to match dimensions of input
b_trans=columnToRow(model_l1_b)
w_transposed=np.transpose(model_l1_W)
resultingOutputs = getOutputs(data,w_transposed,b_trans)
Output:
THE CURRENT I0
[-0.66399151 -0.59143853 0.5230611 -0.52583802 -0.31089544 0.47396523
-0.7301591 -0.21042131 0.92044264 -0.48792791 -1.54127669]
transposed I:
[[-0.66399151 -0.59143853 0.5230611 -0.52583802 -0.31089544 0.47396523
-0.7301591 -0.21042131 0.92044264 -0.48792791 -1.54127669]]
Itrans shape:
(1, 11)
Current biases:
[ 0.04497563 -0.01878226 0.03285328 0.00443657 -0.10408497 0.03982726
-0.07724283]
Current biases shape:
(7,)
B trans: [[ 0.04497563 -0.01878226 0.03285328 0.00443657 -0.10408497 0.03982726
-0.07724283]]
B trans shape: (1, 7)
Current weights:
[[ 0.02534341 0.01163373 -0.20102289 0.23845847 0.20859972 -0.09515963
0.00744185 -0.06694793 -0.03806938 0.02241485 0.34134269]
[ 0.0828636 -0.14711063 0.44623381 0.0095899 0.41908434 -0.25378567
0.35789928 0.21531652 -0.05924326 -0.18556432 0.23026766]
[-0.23547475 -0.18090464 -0.15210266 0.10483326 -0.0182989 0.52936584
0.15671678 -0.64570689 -0.27296376 0.28720504 0.21922119]
[-0.17561196 -0.42502806 -0.34866759 -0.07662395 -0.02361901 -0.10330012
-0.2626377 0.19807351 0.20543958 -0.34499851 0.29347673]
[-0.04404973 -0.31600055 -0.22984107 0.21733086 -0.15065287 0.18301299
0.13399698 0.11884601 0.04380761 -0.03720044 0.0146924 ]
[ 0.25086868 0.15678053 0.30350113 0.13065964 -0.30319506 0.47015968
0.00549904 0.32486886 -0.00331726 0.22858304 0.16789439]
[-0.10196115 -0.03687141 -0.28674102 0.01066647 0.2475083 0.15808311
-0.1452509 0.09170815 -0.14578934 -0.07375327 -0.16524883]]
Transposed weights
[[ 0.02534341 0.0828636 -0.23547475 -0.17561196 -0.04404973 0.25086868
-0.10196115]
[ 0.01163373 -0.14711063 -0.18090464 -0.42502806 -0.31600055 0.15678053
-0.03687141]
[-0.20102289 0.44623381 -0.15210266 -0.34866759 -0.22984107 0.30350113
-0.28674102]
[ 0.23845847 0.0095899 0.10483326 -0.07662395 0.21733086 0.13065964
0.01066647]
[ 0.20859972 0.41908434 -0.0182989 -0.02361901 -0.15065287 -0.30319506
0.2475083 ]
[-0.09515963 -0.25378567 0.52936584 -0.10330012 0.18301299 0.47015968
0.15808311]
[ 0.00744185 0.35789928 0.15671678 -0.2626377 0.13399698 0.00549904
-0.1452509 ]
[-0.06694793 0.21531652 -0.64570689 0.19807351 0.11884601 0.32486886
0.09170815]
[-0.03806938 -0.05924326 -0.27296376 0.20543958 0.04380761 -0.00331726
-0.14578934]
[ 0.02241485 -0.18556432 0.28720504 -0.34499851 -0.03720044 0.22858304
-0.07375327]
[ 0.34134269 0.23026766 0.21922119 0.29347673 0.0146924 0.16789439
-0.16524883]]
wtrans shape:
(11, 7)
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-162-7e8be1d52690> in <module>()
48 #Test calcIndividualOutput
49
---> 50 testOutput= calcIndividualOutput(currI,w_transposed,b_trans)
51 print("Test calcIndividualOutput:\n\n",testOutput,"\n\n")
52 print("Test calcIndividualOutput Shape:\n\n",testOutput.shape,"\n\n")
<ipython-input-162-7e8be1d52690> in calcIndividualOutput(indivInputs, weights, biases)
7 # finds the resulting y values for one set of input data
8 I_transposed= columnToRow(indivInputs)
----> 9 output = np.multiply(I_transposed, weights) + biases
10 return output
11
ValueError: operands could not be broadcast together with shapes (1,11) (11,7)
np.multiply is for multiplying arrays element-wise, but from the dimensions of you data I guess that you are looking for matrix multiplication. To get that use np.dot.
The dot product maps R^n x R^n -> R, that's probably what you want.
If you're coming from Matlab, that's the same difference like A * B and A .* B
I think you are looking for np.matmul(a,b)
This is the row-wise actual multiplication that we actually do in math.
so if a = AxB dimension and b = BxC dimension..then
res = np.matmul(a,b) will have shape AxC..
Related
How to build a vector of marginal probabilities, given a tensor in PyTorch
I have a tensor 'A' of shape [ Dim1: <128>, Dim2: <64>], each element in Dim1 is drawn from a unknown distribution and I need to check if the Dim2 vector has appeared before in the other 128 samples. If it has, the marginal probability of this element is increased by 1 and recorded in another tensor 'B' which is of shape [DimB: <128>]. When the iteration is complete I divide all elements in B by 128 (the number of possibilities) to achieve the weighted delta, therefore the aim is to approximate the true distribution as Dim1 increases in size.
How can this be achieved directly in PyTorch? I attempted it with ordered dictionaries but it's too slow. I'm assuming a method exists to do it straight in PyTorch
my crude way to do it with ordered dictionaries if we have a tensor T1 of shape [Dim1: <6>, Dim2: <3>]:
from collections import OrderedDict
od = OrderedDict()
T1 = torch.Tensor([[2.4, 5.5,1],
[3.44,5.43,1],
[2.4, 5.5,1],
[3.44,8.43,1],
[3.44,5.43,9],
[3.44,5.43,1], ])
print ('T1 shape',T1.shape) # -> T1 shape torch.Size([6, 3])
for i in range(T1.shape[0]):
key = ''.join([ str(int(j)) for j in T1[i].tolist()]) # creates a unique identifier (is there a better way to do this?)
if key in od:
od[key] +=1
key_place_holder = key + str(od[key]) # unique identifier if we found duplicate to keep a 0 in the final tensor
od[key_place_holder] = 0
else:
od[key] = 1
print ('len od',len(od)) # -> len od 6
list_values = [j/len(od) for i,j in od.items()]
T1_marginal_probabilities = torch.Tensor(list_values)
print ('Marginal Probs',T1_marginal_probabilities) # -> Marginal Probs tensor([0.3333, 0.3333, 0.0000, 0.1667, 0.1667, 0.0000])
The final output is as expected, as the probability of [2.4, 5.5,1] and [3.44,5.43,1] are both 2/6, since we have [2.4, 5.5,1] repeated 2 times at position 0 and 2. while [3.44,5.43,1] is repeated in position 1 and 5.
You can use torch.unique and torch.nonzero:
T1 = ...
values, inverse, counts = T1.unique(dim=0, return_inverse=True, return_counts=True)
ps = torch.zeros(inverse.numel())
for i, (v, c) in enumerate(zip(values, counts)):
first_occurence = torch.nonzero(inverse == i)[0].item()
ps[first_occurence] = c
ps /= ps.sum()
Dear jax experts I need your kind help.
Here is a working example (I have follow the advise to simplify my code, although I am not an expert on jax neither on Python to guess what is the heart of the mechanism involved in vmap)
def jax_kernel(rng_key, logpdf, position, log_prob):
key, subkey = jax.random.split(rng_key)
move_proposals = jax.random.normal(key, shape=position.shape)* 0.1
proposal = position + move_proposals
proposal_log_prob = logpdf(proposal)
return proposal, proposal_log_prob
def jax_sampler(rng_key, n_samples, logpdf, initial_position):
def mh_update(i, state):
key, positions, log_prob = state
_, key = jax.random.split(key)
print(f"mh_update: positions[{i-1}]:",jnp.asarray(positions[i-1]))
new_position, new_log_prob = jax_kernel(key,logpdf,positions[i-1],log_prob)
positions=positions.at[i].set(new_position)
return (key, positions, new_log_prob)
# all positions structure should be set before lax.fori_loop
print("initial_position shape:",initial_position.shape)
all_positions = jnp.zeros((n_samples,)+initial_position.shape)
all_positions=all_positions.at[0,0].set(1.)
all_positions=all_positions.at[0,1].set(2.)
all_positions=all_positions.at[0,2].set(2.)
print("all_positions init:",all_positions.shape)
logp = logpdf(all_positions[0])
# use of a for-loop to be able to debug mh_update instead of a jax.fori_loop
initial_state = (rng_key,all_positions, logp)
val = initial_state
for i in range(1, n_samples):
val = mh_update(i, val)
rng_key, all_positions, log_prob = val
# return all the positions of the parameters (n_chains, n_samples, n_dim)
return all_positions
def func(par):
xi = jnp.asarray(sci_stats.uniform.rvs(size=10))
val = xi*par[1]+par[0]
return jnp.sum(jax.scipy.stats.norm.logpdf(x=val,loc=yi,scale=par[2]))
n_dim = 3 # number of parameters ie. (a,b,s)
n_samples = 5 # number of samples per chain
n_chains = 4 # number of MCMC chains
rng_key = jax.random.PRNGKey(42)
rng_keys = jax.random.split(rng_key, n_chains)
initial_position = jnp.ones((n_dim, n_chains))
print("main initial_position shape",initial_position.shape)
run = jax.vmap(jax_sampler, in_axes=(0, None, None, 1), out_axes=0)
all_positions = run(rng_keys,n_samples,lambda p: func(p),initial_position)
print("all_positions:",all_positions)
Then my question concerns the dimension evolution print(f"mh_update: positions[{i-1}]:",jnp.asarray(positions[i-1])). I do not understand why positions[i-1]starts with dimension n_dim and then switches to n_chains x n_dim?
Thanks in advance for your comments?
Here is the complete output:
main initial_position shape (3, 4)
initial_position shape: (3,)
all_positions init: (5, 3)
mh_update: positions[0]: [1. 2. 2.]
mh_update: positions[1]: Traced<ShapedArray(float32[3])>with<BatchTrace(level=1/0)>
with val = DeviceArray([[0.9354116 , 1.7876872 , 1.8443539 ],
[0.9844745 , 2.073029 , 1.9511036 ],
[0.98202926, 2.0109322 , 2.094176 ],
[0.9536771 , 1.9731759 , 2.093319 ]], dtype=float32)
batch_dim = 0
mh_update: positions[2]: Traced<ShapedArray(float32[3])>with<BatchTrace(level=1/0)>
with val = DeviceArray([[1.0606856, 1.6707807, 1.8377957],
[1.0465866, 1.9754674, 1.7009288],
[1.1107644, 2.0142047, 2.190575 ],
[1.0089972, 1.9953227, 1.996874 ]], dtype=float32)
batch_dim = 0
mh_update: positions[3]: Traced<ShapedArray(float32[3])>with<BatchTrace(level=1/0)>
with val = DeviceArray([[1.0731456, 1.644405 , 2.1343162],
[1.0599504, 2.0121546, 1.6867112],
[1.0585173, 1.9661485, 2.1573594],
[1.1213307, 1.9335203, 1.9683584]], dtype=float32)
batch_dim = 0
all_positions: [[[1. 2. 2. ]
[0.9354116 1.7876872 1.8443539 ]
[1.0606856 1.6707807 1.8377957 ]
[1.0731456 1.644405 2.1343162 ]
[1.0921828 1.5742197 2.058759 ]]
[[1. 2. 2. ]
[0.9844745 2.073029 1.9511036 ]
[1.0465866 1.9754674 1.7009288 ]
[1.0599504 2.0121546 1.6867112 ]
[1.0835105 2.0051234 1.4766487 ]]
[[1. 2. 2. ]
[0.98202926 2.0109322 2.094176 ]
[1.1107644 2.0142047 2.190575 ]
[1.0585173 1.9661485 2.1573594 ]
[1.1728328 1.981367 2.180744 ]]
[[1. 2. 2. ]
[0.9536771 1.9731759 2.093319 ]
[1.0089972 1.9953227 1.996874 ]
[1.1213307 1.9335203 1.9683584 ]
[1.1148386 1.9598911 2.1721165 ]]]
In the first iteration, you print a concrete array that you have constructed within a vmapped function. It is a float32 array of shape (3,).
After the first iteration, you've constructed a new array via operations on a vmapped input. When you vmap an input like this, JAX replaces your input array with a tracer that is an abstract representation of your input; the printed value looks like this:
Traced<ShapedArray(float32[3])>with<BatchTrace(level=1/0)>
with val = DeviceArray([[1.0731456, 1.644405 , 2.1343162],
[1.0599504, 2.0121546, 1.6867112],
[1.0585173, 1.9661485, 2.1573594],
[1.1213307, 1.9335203, 1.9683584]], dtype=float32)
The float32[3] indicates that this tracer represents an array of float32 values of shape (3,): that is, it still has the same type and shape as in the first iteration. But in this case it is not a concrete array with three elements, it is a batched tracer representing each iteration of the vmapped input. The power of the vmap transform is that JAX effectively tracks all implied iterations of the vmapped computation in one pass: in the tracer representation, the rows of val effectively show you the intermediate values for all the vmapped iterations.
For more understanding of how JAX tracing works, a good read is How To Think In JAX in the JAX documentation.
In the following code snippet I intend to do the following:
(1) Multiply each element of the identity by the d optimization variable.
(2) Sum a vector of ones to a CVXPY affine expression, which is also a vector of 24 elements.
(3) Create a constraint which compares two vectors element-wise.
import numpy as np
import cvxpy as cp
weights = cp.Variable(5)
d = cp.Variable(1)
meas = np.random.rand(8, 3)
det = np.random.rand(24, 5)
dm = d * np.eye(3) # (1)
beh = np.ones([24, 1]) + cp.reshape((dm # meas.T).T, [24, 1]) # (2)
constrs = [beh == det # weights] #(3)
My questions are:
Q1: Did I code what I wanted?
Q2: At (2), I get the following error:
/usr/lib/python3.8/site-packages/cvxpy/utilities/shape.py in sum_shapes(shapes)
45 # Only allow broadcasting for 0D arrays or summation of scalars.
46 if shape != t and len(squeezed(shape)) != 0 and len(squeezed(t)) != 0:
---> 47 raise ValueError(
48 "Cannot broadcast dimensions " +
49 len(shapes)*" %s" % tuple(shapes))
ValueError: Cannot broadcast dimensions (24, 1) [24, 1]
What exactly does this mean, and how do I fix it?
Q3: When I do det # weights, at (3), I get an Expression(AFFINE, UNKNOWN, (24,)). In the constraint, I'll compare it with beh, which I'm guessing will be an Expression(AFFINE, UNKNOWN, (24, 1)). Will this comparison also bring an issue?
When I started using cvxpy, I also had some trouble making dimensions fit. In my experience, it is a good idea to use arrays with as few dimensions as possible. So if you have a 2-dimensional array where 1 of the dimensions only has length 1, see if you can reduce the dimension. (see below)
The problem in (2) is solved when you changed the brackets you use when reshaping the cvxpy expression to (24,1), like this:
beh = np.ones([24, 1]) + cp.reshape((dm # meas.T).T, (24, 1)) # (2)
You could also avoid your problem by simply doing:
beh = 1 + cp.reshape((dm # meas.T).T, (24, 1)) # (2)
which will do the same: add 1 to each entry of the cvxpy array.
After this is done, you will have a problem with your final line: "ValueError: Cannot broadcast dimensions (24, 1) (24,)"
This can be remedied by making beh of the dimension (24, ) too (reduce the dimension will solve your problems here, as mentioned before). The full working code would be:
import numpy as np
import cvxpy as cp
weights = cp.Variable(5)
d = cp.Variable(1)
meas = np.random.rand(8, 3)
det = np.random.rand(24, 5)
dm = d * np.eye(3) # (1)
beh = 1 + cp.reshape((dm # meas.T).T, (24, )) # (2)
constrs = [beh == det # weights] #(3)
Hope this helps!
I have created the following Python program, which, as far as I understand CTC, should be a valid CTC-based model, as well as training data. The best documentation I can find is CNTK_208_Speech_CTC Tutorial, which is what I've based this on. The program is as simple as I could make it, and it relies only on numpy and CNTK, and generates data itself.
When I run this, I get the following error:
Validating --> ForwardBackward2850 = ForwardBackward (LabelsToGraph2847, StableSigmoid2703) : [5 x labelAxis1], [5 x inputAxis1] -> []
RuntimeError: The Matrix dimension in the ForwardBackwardNode operation does not match.
This seems to be the same issue from this ticket: https://github.com/Microsoft/CNTK/issues/2156
Here is the Python program:
# cntk_ctc_hello_world.py
#
# This is a "hello world" example of using CTC (Connectionist Temporal Classification) with CNTK.
#
# The input is a sequence of vectors of size 17. We use 17 because it's easy to spot that number in
# error messages. The output is a string of codes, each code being one of 4 possible characters from
# our alphabet that we'll refer to here as "ABCD", although they're actually just represented
# by the numbers 0..3, which is typical for classification systems. To make the setup of training data
# trivial, we assign the first four elements of our 17-dimension input vector to the four characters
# of our alphabet, so that the matching is:
# 10000000000000000 A
# 01000000000000000 B
# 00100000000000000 C
# 00010000000000000 D
# In our input sequences, we repeat each code three to five times, followed by three to five codes
# containing random noise. Whether it's repeated 3,4, or 5 times, is random for each code and each
# spacer. When we emit one of our codes, we fill the first 4 values with the code, and the remaining
# 13 values with random noise.
# For example:
# Input: AAA-----CCCC---DDDDD
# Output: ACD
import cntk as C
import numpy as np
import random
import sys
InputDim = 17
NumClasses = 4 # A,B,C,D
MinibatchSize = 100
MinibatchPerEpoch = 50
NumEpochs = 10
MaxOutputSeqLen = 10 # ABCDABCDAB
inputAxis = C.Axis.new_unique_dynamic_axis('inputAxis')
labelAxis = C.Axis.new_unique_dynamic_axis('labelAxis')
inputVar = C.sequence.input_variable((InputDim), sequence_axis=inputAxis, name="input")
labelVar = C.sequence.input_variable((NumClasses+1), sequence_axis=labelAxis, name="labels")
# Construct an LSTM-based model that will perform the classification
with C.default_options(activation=C.sigmoid):
classifier = C.layers.Sequential([
C.layers.For(range(3), lambda: C.layers.Recurrence(C.layers.LSTM(128))),
C.layers.Dense(NumClasses + 1)
])(inputVar)
criteria = C.forward_backward(C.labels_to_graph(labelVar), classifier, blankTokenId=NumClasses, delayConstraint=3)
err = C.edit_distance_error(classifier, labelVar, squashInputs=True, tokensToIgnore=[NumClasses])
lr = C.learning_rate_schedule([(3, .01), (1,.001)], C.UnitType.sample)
mm = C.momentum_schedule([(1000, 0.9), (0, 0.99)], MinibatchSize)
learner = C.momentum_sgd(classifier.parameters, lr, mm)
trainer = C.Trainer(classifier, (criteria, err), learner)
# Return a numpy array of 17 elements, for this code
def make_code(code):
a = np.zeros(NumClasses) # 0,0,0,0
v = np.random.rand(InputDim - NumClasses) # 13x random
a = np.concatenate((a, v))
a[code] = 1
return a
def make_noise_code():
return np.random.rand(InputDim)
def make_onehot(code):
v = np.zeros(NumClasses+1)
v[code] = 1
return v
def gen_batch():
x_batch = []
y_batch = []
for mb in range(MinibatchSize):
yLen = random.randint(1, MaxOutputSeqLen)
x = []
y = []
for i in range(yLen):
code = random.randint(0,3)
y.append(make_onehot(code))
xLen = random.randint(3,5) # Input is 3 to 5 repetitions of the code
for j in range(xLen):
x.append(make_code(code))
spacerLen = random.randint(3,5) # Spacer is 3 to 5 repetitions of noise
for j in range(spacerLen):
x.append(make_noise_code())
x_batch.append(np.array(x, dtype='float32'))
y_batch.append(np.array(y, dtype='float32'))
return x_batch, y_batch
#######################################################################################
# Dump first X/Y training pair from minibatch
#x, y = gen_batch()
#print("\nx sequence of first sample of minibatch:\n", x[0])
#print("\ny sequence of first sample of minibatch:\n", y[0])
#######################################################################################
progress_printer = C.logging.progress_print.ProgressPrinter(tag='Training', num_epochs=NumEpochs)
for epoch in range(NumEpochs):
for mb in range(MinibatchPerEpoch):
x_batch, y_batch = gen_batch()
trainer.train_minibatch({inputVar: x_batch, labelVar: y_batch})
progress_printer.epoch_summary(with_metric=True)
For those who are facing this error, there are two points to take note of:
(1) The data provided to labels sequence tensor to labels_to_graph must have the same sequence length as the data coming out from network output at runtime.
(2) If during the model building you change the dynamic sequence axis of input sequence tensor (e.g. stride in sequential axis), then you must call reconcile_dynamic_axes on your labels sequence tensor with the network_output as the second argument to the function. This tells CNTK that labels have the same dynamic axis as the network.
Adhering to these 2 points will allow forward_backward to run.
I need to create a generator for my data to pass into my RNN training function. I have a list of patient samples, where each sample is a time series of length ni (which varies, annoyingly) in three dimensions, and I want to create batches of data where each sample in a batch only belongs to a single patient but each batch may contain multiple patient samples. Doing it this way should maximise the number of samples I can train using with no consequences as my RNN is not stateful. At first I had the following function
def dataIterator(rawDataList, config):
batchSize, nSteps = config.batchSize, config.nSteps
for rawData in rawDataList:
dataLen, dataWidth = rawData.shape
batchLen = dataLen // batchSize
data = np.zeros([batchSize, batchLen, dataWidth], dtype=np.float32)
for i in xrange(batchSize):
data[i] = rawData[batchLen*i:batchLen*(i+1), :]
epochSize = (batchLen - 1) // nSteps
if epochSize == 0:
raise ValueError('epoch_size == 0')
for i in xrange(epochSize):
x = data[:, i*nSteps:(i+1)*nSteps, :]
y = data[:, i*nSteps+1:(i+1)*nSteps+1, :]
yield (x, y)
However this trims each of the patient samples in order to fit the batch size. So I want something that creates all possible batches, including the undersized one at the end. However my unfamiliarity with generators has left me pretty confused. So far I've worked out it's going to have to use modulo aritmetic, but exactly how I'm not sure, so I've only got to this point:
def dataIterator(data, batchSize=batchSize, nSteps=nSteps, nDimensions=3):
nTimePoints = sum([len(x) for x in data])
totalBatchLen = 1+(nTimePoints-1)//batchSize
newData = np.zeros([batchSize, totalBatchLen, nDimensions])
for i in xrange(batchSize):
...
EDIT
Here's a short example to show how I would solve the problem without using generators
import numpy as np
np.random.seed(42)
nPatients = 3
tsLength = 5
nDimensions = 3
rnnTSLength = 3
batchSize = 3
inputData = np.random.random((nPatients, tsLength, nDimensions))
inputData[1, :, :] *= 10
inputData[2, :, :] *= 100
outputData = []
for i in xrange(tsLength-rnnTSLength):
outputData.append(inputData[0, i:i+rnnTSLength, :])
for i in xrange(tsLength-rnnTSLength):
outputData.append(inputData[1, i:i+rnnTSLength, :])
for i in xrange(tsLength-rnnTSLength):
outputData.append(inputData[2, i:i+rnnTSLength, :])
temp1 = np.array(outputData[:3])
temp2 = np.array(outputData[3:])
npOutput = np.array((temp1, temp2))
print npOutput
Which produces:
[[[[ 3.74540119e-01 9.50714306e-01 7.31993942e-01]
[ 5.98658484e-01 1.56018640e-01 1.55994520e-01]
[ 5.80836122e-02 8.66176146e-01 6.01115012e-01]]
[[ 5.98658484e-01 1.56018640e-01 1.55994520e-01]
[ 5.80836122e-02 8.66176146e-01 6.01115012e-01]
[ 7.08072578e-01 2.05844943e-02 9.69909852e-01]]
[[ 1.83404510e+00 3.04242243e+00 5.24756432e+00]
[ 4.31945019e+00 2.91229140e+00 6.11852895e+00]
[ 1.39493861e+00 2.92144649e+00 3.66361843e+00]]]
[[[ 4.31945019e+00 2.91229140e+00 6.11852895e+00]
[ 1.39493861e+00 2.92144649e+00 3.66361843e+00]
[ 4.56069984e+00 7.85175961e+00 1.99673782e+00]]
[[ 6.07544852e+01 1.70524124e+01 6.50515930e+00]
[ 9.48885537e+01 9.65632033e+01 8.08397348e+01]
[ 3.04613769e+01 9.76721140e+00 6.84233027e+01]]
[[ 9.48885537e+01 9.65632033e+01 8.08397348e+01]
[ 3.04613769e+01 9.76721140e+00 6.84233027e+01]
[ 4.40152494e+01 1.22038235e+01 4.95176910e+01]]]]
Which as you can see has two batches of size three, both of which contain two different 'patients' in them, but the time series for each 'patient' do not overlap.
It's not exactly clear what you are looking for. A small sample of input and desired output would help. Nevertheless, I'll take a stab at what I think you are asking:
def dataIterator(data, batchSize=batchSize):
for patient_data in data:
for n in range(0, len(patient_data), batchSize):
yield patient_data[n:n+batchSize]