Develop Reference

timm: AssertionError: Batch size should be even when using this

I am implementing simple example for creating a model from scratch using timm. My batch is even but still i am getting an error and I am following below tutorials
https://gist.github.com/Chris-hughes10/a9e5ec2cd7e7736c651bf89b5484b4a9
import argparse
from pathlib import Path
import timm
import timm.data
import timm.loss
import timm.optim
import timm.utils
import torch
import torchmetrics
from timm.scheduler import CosineLRScheduler
from pytorch_accelerated.callbacks import SaveBestModelCallback
from pytorch_accelerated.trainer import Trainer, DEFAULT_CALLBACKS
def create_datasets(image_size, data_mean, data_std, train_path, val_path):
train_transforms = timm.data.create_transform(
input_size=image_size,
is_training=True,
mean=data_mean,
std=data_std,
auto_augment="rand-m7-mstd0.5-inc1",
)
eval_transforms = timm.data.create_transform(
input_size=image_size, mean=data_mean, std=data_std
)
train_dataset = timm.data.dataset.ImageDataset(
train_path, transform=train_transforms
)
eval_dataset = timm.data.dataset.ImageDataset(val_path, transform=eval_transforms)
return train_dataset, eval_dataset
class TimmMixupTrainer(Trainer):
def __init__(self, eval_loss_fn, mixup_args, num_classes, *args, **kwargs):
super().__init__(*args, **kwargs)
self.eval_loss_fn = eval_loss_fn
self.num_updates = None
self.mixup_fn = timm.data.Mixup(**mixup_args)
self.accuracy = torchmetrics.Accuracy(num_classes=num_classes)
self.ema_accuracy = torchmetrics.Accuracy(num_classes=num_classes)
self.ema_model = None
def create_scheduler(self):
return timm.scheduler.CosineLRScheduler(
self.optimizer,
t_initial=self.run_config.num_epochs,
cycle_decay=0.5,
lr_min=1e-6,
t_in_epochs=True,
warmup_t=3,
warmup_lr_init=1e-4,
cycle_limit=1,
)
def training_run_start(self):
# Model EMA requires the model without a DDP wrapper and before sync batchnorm conversion
self.ema_model = timm.utils.ModelEmaV2(
self._accelerator.unwrap_model(self.model), decay=0.9
)
if self.run_config.is_distributed:
self.model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(self.model)
def train_epoch_start(self):
super().train_epoch_start()
self.num_updates = self.run_history.current_epoch * len(self._train_dataloader)
def calculate_train_batch_loss(self, batch):
xb, yb = batch
mixup_xb, mixup_yb = self.mixup_fn(xb, yb)
return super().calculate_train_batch_loss((mixup_xb, mixup_yb))
def train_epoch_end(
self,
):
self.ema_model.update(self.model)
self.ema_model.eval()
if hasattr(self.optimizer, "sync_lookahead"):
self.optimizer.sync_lookahead()
def scheduler_step(self):
self.num_updates += 1
if self.scheduler is not None:
self.scheduler.step_update(num_updates=self.num_updates)
def calculate_eval_batch_loss(self, batch):
with torch.no_grad():
xb, yb = batch
outputs = self.model(xb)
val_loss = self.eval_loss_fn(outputs, yb)
self.accuracy.update(outputs.argmax(-1), yb)
ema_model_preds = self.ema_model.module(xb).argmax(-1)
self.ema_accuracy.update(ema_model_preds, yb)
return {"loss": val_loss, "model_outputs": outputs, "batch_size": xb.size(0)}
def eval_epoch_end(self):
super().eval_epoch_end()
if self.scheduler is not None:
self.scheduler.step(self.run_history.current_epoch + 1)
self.run_history.update_metric("accuracy", self.accuracy.compute().cpu())
self.run_history.update_metric(
"ema_model_accuracy", self.ema_accuracy.compute().cpu()
)
self.accuracy.reset()
self.ema_accuracy.reset()
def main(data_path):
# Set training arguments, hardcoded here for clarity
image_size = (224, 224)
lr = 5e-3
smoothing = 0.1
mixup = 0.2
cutmix = 1.0
batch_size = 32
bce_target_thresh = 0.2
num_epochs = 40
data_path = Path(data_path)
train_path = data_path / "train"
val_path = data_path / "val"
num_classes = len(list(train_path.iterdir()))
mixup_args = dict(
mixup_alpha=mixup,
cutmix_alpha=cutmix,
label_smoothing=smoothing,
num_classes=num_classes,
)
# Create model using timm
model = timm.create_model(
"resnet50d", pretrained=False, num_classes=num_classes, drop_path_rate=0.05
)
# Load data config associated with the model to use in data augmentation pipeline
data_config = timm.data.resolve_data_config({}, model=model, verbose=True)
data_mean = data_config["mean"]
data_std = data_config["std"]
# Create training and validation datasets
train_dataset, eval_dataset = create_datasets(
train_path=train_path,
val_path=val_path,
image_size=image_size,
data_mean=data_mean,
data_std=data_std,
)
# Create optimizer
optimizer = timm.optim.create_optimizer_v2(
model, opt="lookahead_AdamW", lr=lr, weight_decay=0.01
)
# As we are using Mixup, we can use BCE during training and CE for evaluation
train_loss_fn = timm.loss.BinaryCrossEntropy(
target_threshold=bce_target_thresh, smoothing=smoothing
)
validate_loss_fn = torch.nn.CrossEntropyLoss()
# Create trainer and start training
trainer = TimmMixupTrainer(
model=model,
optimizer=optimizer,
loss_func=train_loss_fn,
eval_loss_fn=validate_loss_fn,
mixup_args=mixup_args,
num_classes=num_classes,
callbacks=[
*DEFAULT_CALLBACKS,
SaveBestModelCallback(watch_metric="accuracy", greater_is_better=True),
],
)
trainer.train(
per_device_batch_size=batch_size,
train_dataset=train_dataset,
eval_dataset=eval_dataset,
num_epochs=num_epochs,
create_scheduler_fn=trainer.create_scheduler,
)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple example of training script using timm.")
parser.add_argument("--data_dir", required=True, help="The data folder on disk.")
args = parser.parse_args()
main(args.data_dir)
Traceback
100%|█████████▉| 333/334 [00:37<00:00, 9.09it/s]Traceback (most recent call last):
File "/home/cvpr/PycharmProjects/timm_tutorials/scratch_model.py", line 201, in <module>
main(args.data_dir)
File "/home/cvpr/PycharmProjects/timm_tutorials/scratch_model.py", line 188, in main
trainer.train(
File "/home/cvpr/anaconda3/envs/timm_tutorials/lib/python3.8/site-packages/pytorch_accelerated/trainer.py", line 437, in train
self._run_training()
File "/home/cvpr/anaconda3/envs/timm_tutorials/lib/python3.8/site-packages/pytorch_accelerated/trainer.py", line 641, in _run_training
self._run_train_epoch(self._train_dataloader)
File "/home/cvpr/anaconda3/envs/timm_tutorials/lib/python3.8/site-packages/pytorch_accelerated/trainer.py", line 704, in _run_train_epoch
batch_output = self.calculate_train_batch_loss(batch)
File "/home/cvpr/PycharmProjects/timm_tutorials/scratch_model.py", line 78, in calculate_train_batch_loss
mixup_xb, mixup_yb = self.mixup_fn(xb, yb)
File "/home/cvpr/anaconda3/envs/timm_tutorials/lib/python3.8/site-packages/timm/data/mixup.py", line 210, in __call__
assert len(x) % 2 == 0, 'Batch size should be even when using this'
AssertionError: Batch size should be even when using this
100%|█████████▉| 333/334 [00:37<00:00, 8.93it/s]

As you can see the model works perfectly until the last batch of the epoch. It is because for the final batch, the loader get the remaining images and put them together in this batch. Unfortunately this final batch seems to have odd size.

BertTokenizer error ValueError: Input nan is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers

import pandas as pd
from sklearn.model_selection import train_test_split
# read text data
df = pd.read_csv('E:/bert4keras-master/resume_data/111.txt', header=None,encoding='utf-8', sep='\t',names=['label', 'sentence'])
print(df)
# split text data
train, valid_test = train_test_split(df, test_size=0.3, shuffle=True, random_state=123, stratify=df['label'])
print(valid_test.head)
valid, test = train_test_split(valid_test, test_size=0.5, shuffle=True, random_state=123, stratify=valid_test['label'])
train.reset_index(drop=True, inplace=True)
valid.reset_index(drop=True, inplace=True)
test.reset_index(drop=True, inplace=True)
class CreateDataset(Dataset):
def __init__(self, X, y, tokenizer, max_len):
self.X = X
self.y = y
self.tokenizer = tokenizer
self.max_len = max_len
def __len__(self): # len(Dataset)
return len(self.y)
def __getitem__(self, index): # Dataset[index]
text = self.X[index]
inputs = self.tokenizer.encode_plus(
text,
add_special_tokens=True,
max_length=self.max_len,
pad_to_max_length=True
)
ids = inputs['input_ids']
mask = inputs['attention_mask']
return {
'ids': torch.LongTensor(ids),
'mask': torch.LongTensor(mask),
'labels': torch.Tensor(self.y[index])
}
# label one-hot
y_train = pd.get_dummies(train, columns=['label'])[['label_Exp','label_PI','label_Sum','label_Edu', 'label_QC', 'label_Skill', 'label_Obj']].values
y_valid = pd.get_dummies(valid, columns=['label'])[['label_Exp','label_PI','label_Sum','label_Edu', 'label_QC', 'label_Skill', 'label_Obj']].values
y_test = pd.get_dummies(test, columns=['label'])[['label_Exp','label_PI','label_Sum','label_Edu', 'label_QC', 'label_Skill', 'label_Obj']].values
# make dataset
max_len = 256
tokenizer = BertTokenizer.from_pretrained('E:/bert4keras-master/pytorch_bert_large/')
dataset_train = CreateDataset(train['sentence'], y_train, tokenizer, max_len)
dataset_valid = CreateDataset(valid['sentence'], y_valid, tokenizer, max_len)
dataset_test = CreateDataset(test['sentence'], y_test, tokenizer, max_len)
# dataloader
dataloader_train = DataLoader(dataset_train, batch_size=batch_size, shuffle=True)
dataloader_valid = DataLoader(dataset_valid, batch_size=len(dataset_valid), shuffle=False)
and I get this error when I try to train model log = train_model(dataset_train, dataset_valid, BATCH_SIZE, model, criterion, optimizer, NUM_EPOCHS, device=device)
ERROR
>>> log = train_model(dataset_train, dataset_valid, BATCH_SIZE, model, criterion, optimizer, NUM_EPOCHS, device=device)
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "<stdin>", line 29, in train_model
File "<stdin>", line 8, in calculate_loss_and_accuracy
File "E:\anaconda3\envs\py38pytorch\lib\site-packages\torch\utils\data\dataloader.py", line 521, in __next__
data = self._next_data()
File "E:\anaconda3\envs\py38pytorch\lib\site-packages\torch\utils\data\dataloader.py", line 561, in _next_data
data = self._dataset_fetcher.fetch(index) # may raise StopIteration
File "E:\anaconda3\envs\py38pytorch\lib\site-packages\torch\utils\data\_utils\fetch.py", line 49, in fetch
data = [self.dataset[idx] for idx in possibly_batched_index]
File "E:\anaconda3\envs\py38pytorch\lib\site-packages\torch\utils\data\_utils\fetch.py", line 49, in <listcomp>
data = [self.dataset[idx] for idx in possibly_batched_index]
File "<stdin>", line 11, in __getitem__
File "E:\anaconda3\envs\py38pytorch\lib\site-packages\transformers\tokenization_utils_base.py", line 2556, in encode_plus
return self._encode_plus(
File "E:\anaconda3\envs\py38pytorch\lib\site-packages\transformers\tokenization_utils.py", line 647, in _encode_plus
first_ids = get_input_ids(text)
File "E:\anaconda3\envs\py38pytorch\lib\site-packages\transformers\tokenization_utils.py", line 634, in get_input_ids
raise ValueError(
ValueError: Input nan is not valid. Should be a string, a list/tuple of strings or a list/tuple of integers.
any suggestions please Thank you very much

yoy should debug in
inputs = self.tokenizer.encode_plus(
text,
add_special_tokens=True,
max_length=self.max_len,
pad_to_max_length=True
)
'text' should string or list string...

Keras memory allocation issues

I have been trying to build a deep learning model, using Keras as API with tensorflow 2.2 and cuda 102.89. My dataset is "relatively big" (27500 400x400 images) and so I've tried using both keras.utils.sequence (here) and tf.keras.preprocessing.image.ImageDataGenerator (here) to fit these images into the memory using batches. Yet, neither of these worked so far, I must be not using it correctly but I can't see what's wrong with my code.
Using the keras.utils.sequence:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from tensorflow import keras
class BatchGenerator(keras.utils.Sequence):
def __init__(self, inputs, targets, batch_size=32):
self.inputs = inputs
self.targets = targets
self.batch_size = batch_size
def __len__(self):
return len(self.targets) // self.batch_size
def __getitem__(self, idx):
i = idx * self.batch_size
x = self.inputs[i : i + self.batch_size]
y = self.targets[i : i + self.batch_size]
return x, y
# Load images and labels with normalization and one hot encoding
# images: np.array, (27512, 480, 480, 3); labels: np.array, (27512, 480, 480, 9); nb_class = 9
images, labels, nb_class = data_loader('../Data/Images', '../Data/labels')
# Creation of the 0.2 validation split
train = list(range(images.shape[0]//5, images.shape[0]))
test = list(range(0, images.shape[0]//5))
# Batch generation
train_gen = BatchGenerator(images[train], labels[train], batch_size, data_aug)
valid_gen = BatchGenerator(images[test], labels[test], batch_size, data_aug)
# Model fitting
history = model.fit(train_gen, epochs=nb_epoch, verbose=1,
validation_data=valid_gen, shuffle=False, callbacks=callbacks)
Using ImageDataGeneratorlink and flow link:
from keras.preprocessing.image import ImageDataGenerator
# Load images and labels with normalization and one hot encoding
# images: np.array, (27512, 480, 480, 3); labels: np.array, (27512, 480, 480, 9); nb_class = 9
images, labels, nb_class = data_loader('../Data/Images', '../Data/labels')
# Batch generation with data augmentation instance
datagen = ImageDataGenerator(
vertical_flip=True,
horizontal_flip=True,
preprocessing_function=augmentations_color,
samplewise_center=True,
samplewise_std_normalization=True,
validation_split=0.2
)
# Batch generation application
train_gen = datagen.flow(images, labels, batch_size=16,
shuffle=True, subset='training')
valid_gen = datagen.flow(images, labels, batch_size=16,
shuffle=True, subset='validation')
# Model fitting
history = model.fit(train_gen, epochs=nb_epoch, verbose=1,
validation_data=valid_gen, shuffle=False, callbacks=callbacks)
In both cases, the code crashes during BatchGenerator and never reach the model fitting part complaining:
Traceback (most recent call last):
File "main.py", line 190, in <module>
conf_file=train_set.cfg_path)
File "main.py", line 92, in main
shuffle=True, subset='training')
File "/hpc_htom/kjam268/Virtual_ENV/HistoTAG/lib/python3.6/site-packages/keras_preprocessing/image/image_data_generator.py", line 434, in flow
dtype=self.dtype
File "/hpc_htom/kjam268/Virtual_ENV/HistoTAG/lib/python3.6/site-packages/keras_preprocessing/image/numpy_array_iterator.py", line 103, in __init__
np.unique(y[split_idx:]))):
File "<__array_function__ internals>", line 6, in unique
File "/hpc_htom/kjam268/Virtual_ENV/HistoTAG/lib/python3.6/site-packages/numpy/lib/arraysetops.py", line 261, in unique
ret = _unique1d(ar, return_index, return_inverse, return_counts)
File "/hpc_htom/kjam268/Virtual_ENV/HistoTAG/lib/python3.6/site-packages/numpy/lib/arraysetops.py", line 314, in _unique1d
ar = np.asanyarray(ar).flatten()
numpy.core._exceptions.MemoryError: Unable to allocate 189. GiB for an array with shape (50711040000,) and data type float32
I have trimmed the code to the essential but I can post all of it if needed. I tried both scenarios with different batch as small as 1 without improvements.
Any ideas of what I could try to be able to train my model ?
Thank you

You can follow this official instruction Image segmentation with a U-Net-like architecture. Here is the most relevant coding part for your case.
Generator
class BatchGenerator(keras.utils.Sequence):
"""Helper to iterate over the data (as Numpy arrays)."""
def __init__(self, batch_size, img_size, input_img_paths, target_img_paths):
self.batch_size = batch_size
self.img_size = img_size
self.input_img_paths = input_img_paths
self.target_img_paths = target_img_paths
def __len__(self):
return len(self.target_img_paths) // self.batch_size
def __getitem__(self, idx):
"""Returns tuple (input, target) correspond to batch #idx."""
i = idx * self.batch_size
batch_input_img_paths = self.input_img_paths[i : i + self.batch_size]
batch_target_img_paths = self.target_img_paths[i : i + self.batch_size]
x = np.zeros((self.batch_size,) + self.img_size + (3,), dtype="float32")
for j, path in enumerate(batch_input_img_paths):
img = load_img(path, target_size=self.img_size)
x[j] = img
y = np.zeros((self.batch_size,) + self.img_size + (1,), dtype="uint8")
for j, path in enumerate(batch_target_img_paths):
img = load_img(path, target_size=self.img_size, color_mode="grayscale")
y[j] = np.expand_dims(img, 2)
# Ground truth labels are 1, 2, 3. Subtract one to make them 0, 1, 2:
y[j] -= 1
return x, y
DataSet
input_dir = "images/"
target_dir = "annotations/trimaps/"
batch_size = 32
input_img_paths = sorted(
[
os.path.join(input_dir, fname)
for fname in os.listdir(input_dir)
if fname.endswith(".jpg")
]
)
target_img_paths = sorted(
[
os.path.join(target_dir, fname)
for fname in os.listdir(target_dir)
if fname.endswith(".png") and not fname.startswith(".")
]
)
print("Number of samples:", len(input_img_paths))
for input_path, target_path in zip(input_img_paths[:10], target_img_paths[:10]):
print(input_path, "|", target_path)
Data Generator
# Split our img paths into a training and a validation set
val_samples = 1000
random.Random(1337).shuffle(input_img_paths)
random.Random(1337).shuffle(target_img_paths)
train_input_img_paths = input_img_paths[:-val_samples]
train_target_img_paths = target_img_paths[:-val_samples]
val_input_img_paths = input_img_paths[-val_samples:]
val_target_img_paths = target_img_paths[-val_samples:]
# Instantiate data Sequences for each split
train_gen = BatchGenerator(
batch_size, img_size, train_input_img_paths, train_target_img_paths
)
val_gen = BatchGenerator(batch_size, img_size, val_input_img_paths,
val_target_img_paths)

TypeError('No suitable SharedVariable constructor could be found.') Using lasagne and theano

Hi i'm currently learning coding with python and have been following a tutorial series which has helped me make the code i will show below. Apologies for it being so long but I cannot pinpoint the line of code which is causing this error. I have removed a lot of the commenting to reduce the amount of code posted.
import numpy as np
import urllib.request
import os
import gzip
import lasagne
import theano
import theano.tensor as T
def load_dataset():
def download(filename, source="http://yann.lecun.com/exdb/mnist/"):
print("downloading:", filename)
urllib.request.urlretrieve(source+filename, filename)
def load_mnist_images(filename):
if not os.path.exists(filename):
download(filename)
with gzip.open(filename, "rb") as f:
data = np.frombuffer(f.read(), np.uint8, offset= 16)
data = data.reshape(-1, 1, 28, 28)
return data / np.float32(256)
def load_mnist_labels(filename):
if not os.path.exists(filename):
download(filename)
with gzip.open(filename, "rb") as f:
data = np.frombuffer(f.read(), np.uint8, offset= 8)
return data
x_train = load_mnist_images("train-images-idx3-ubyte.gz")
y_train = load_mnist_labels("train-labels-idx1-ubyte.gz")
x_test = load_mnist_images("t10k-images-idx3-ubyte.gz")
y_test = load_mnist_labels("t10k-labels-idx1-ubyte.gz")
return x_train, y_train, x_test, y_test
x_train, y_train, x_test, y_test = load_dataset()
###### creating the handwriting digit recognition code ######
def build_nn(input_var = None):
l_in = lasagne.layers.InputLayer(shape=(None,1,28,28), input_var=input_var)
l_in_drop = lasagne.layers.DropoutLayer(l_in, p=0.2)
l_hid1 = lasagne.layers.DenseLayer(l_in_drop, num_units= 800,
nonlinearity= lasagne.nonlinearities.rectify,
W= lasagne.init.GlorotUniform())
l_hid1_drop = lasagne.layers.DropoutLayer(l_hid1, p=0.5)
l_hid2 = lasagne.layers.DenseLayer(l_hid1_drop, num_units= 800,
nonlinearity= lasagne.nonlinearities.rectify,
W= lasagne.init.GlorotUniform())
l_hid2_drop = lasagne.layers.DropoutLayer(l_hid2, p=0.5)
l_out = lasagne.layers.DenseLayer(l_hid2_drop, num_units=10,
nonlinearity= lasagne.nonlinearities.softmax)
return l_out
input_var = T.tensor4("inputs") # an empty 4d array
target_var = T.ivector("targets") # an empty 1d int array to represent the labels
network = build_nn(input_var) # call the func that initializes the neural network
prediction = lasagne.layers.get_output(network)
loss = lasagne.objectives.categorical_crossentropy(prediction, target_var)
loss = loss.mean()
params = lasagne.layers.get_all_params(network, trainable=True)
updates = lasagne.updates.nesterov_momentum(loss, params, learning_rate=0.01, momentum=0.9)
train_fn = theano.function([input_var, target_var], loss, updates= updates)
num_training_steps = 10
for step in range(num_training_steps):
train_err = train_fn(x_train, y_train)
print("current training step is " + str(step))
The error that's stopping this code is this:
Traceback (most recent call last):
File "C:\Users\Admin\.vscode\Practice codes\machine learning\deep learning\deep learning.py", line 125, in <module>
network = build_nn(input_var) # call the func that initializes the neural network
File "C:\Users\Admin\.vscode\Practice codes\machine learning\deep learning\deep learning.py", line 95, in build_nn
l_hid1 = lasagne.layers.DenseLayer(l_in_drop, num_units= 800,
File "C:\Users\Admin\AppData\Roaming\Python\Python38\site-packages\lasagne\layers\dense.py", line 103, in __init__
self.W = self.add_param(W, (num_inputs, num_units), name="W")
File "C:\Users\Admin\AppData\Roaming\Python\Python38\site-packages\lasagne\layers\base.py", line 234, in add_param
param = utils.create_param(spec, shape, name)
File "C:\Users\Admin\AppData\Roaming\Python\Python38\site-packages\lasagne\utils.py", line 393, in create_param
spec = theano.shared(spec, broadcastable=bcast)
File "C:\Users\Admin\AppData\Roaming\Python\Python38\site-packages\theano\compile\sharedvalue.py", line 284, in shared
raise TypeError('No suitable SharedVariable constructor could be found.'
TypeError: No suitable SharedVariable constructor could be found. Are you sure all kwargs are supported? We do not support the parameter dtype or type. value="[[ 0.04638761 -0.02959769 0.02330909 ... 0.01545383 0.04763002
0.05265676]
[ 0.02095251 -0.05393376 -0.04289599 ... -0.02409102 0.02824548
-0.00327342]
[ 0.02908951 -0.02853872 -0.05450716 ... -0.02296509 0.02495853
0.02486875]
...
[-0.03704383 0.0286258 0.01158947 ... -0.02583007 -0.04925423
-0.0470493 ]
[ 0.03230407 -0.00246115 -0.05074456 ... 0.00299953 0.01883504
0.01312843]
[-0.05762409 -0.05119916 -0.02820581 ... -0.05675326 0.00458562
0.04403118]]". parameters="{'broadcastable': (False, False)}"
If it helps I'm using python 3.8 - lasagne 0.2.dev1 - theano 1.0.5.
Any help would be greatly appreciated, any questions feel free to ask.
Thanks in advance

'bert-base-multilingual-uncased' dataloader RuntimeError : stack expects each tensor to be equal size

I am a begineer in nlp, as I was giving this competition https://www.kaggle.com/c/contradictory-my-dear-watson I am using the model 'bert-base-multilingual-uncased' and using BERT tokenizer from the same. I am also using kaggle tpu. This is the custom dataloader I created.
class SherlockDataset(torch.utils.data.Dataset):
def __init__(self,premise,hypothesis,tokenizer,max_len,target = None):
super(SherlockDataset,self).__init__()
self.premise = premise
self.hypothesis = hypothesis
self.tokenizer = tokenizer
self.max_len = max_len
self.target = target
def __len__(self):
return len(self.premise)
def __getitem__(self,item):
sen1 = str(self.premise[item])
sen2 = str(self.hypothesis[item])
encode_dict = self.tokenizer.encode_plus(sen1,
sen2,
add_special_tokens = True,
max_len = self.max_len,
pad_to_max_len = True,
return_attention_mask = True,
return_tensors = 'pt'
)
input_ids = encode_dict["input_ids"][0]
token_type_ids = encode_dict["token_type_ids"][0]
att_mask = encode_dict["attention_mask"][0]
if self.target is not None:
sample = {
"input_ids":input_ids,
"token_type_ids":token_type_ids,
"att_mask":att_mask,
"targets": self.target[item]
}
else:
sample = {
"input_ids":input_ids,
"token_type_ids":token_type_ids,
"att_mask":att_mask
}
return sample
and during the time of loading data in dataloader
def train_fn(model,dataloader,optimizer,criterion,scheduler = None):
model.train()
print("train")
for idx, sample in enumerate(dataloader):
'''
input_ids = sample["input_ids"].to(config.DEVICE)
token_type_ids = sample["token_type_ids"].to(config.DEVICE)
att_mask = sample["att_mask"].to(config.DEVICE)
targets = sample["targets"].to(config.DEVICE)
'''
print("train_out")
input_ids = sample[0].to(config.DEVICE)
token_type_ids = sample[1].to(config.DEVICE)
att_mask = sample[2].to(config.DEVICE)
targets = sample[3].to(config.DEVICE)
optimizer.zero_grad()
output = model(input_ids,token_type_ids,att_mask)
output = np.argmax(output,axis = 1)
loss = criterion(outputs,targets)
accuracy = accuracy_score(output,targets)
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(),1.0)
xm.optimizer_step(optimizer, barrier=True)
if scheduler is not None:
scheduler.step()
if idx%50==0:
print(f"idx : {idx}, TRAIN LOSS : {loss}")
I am facing this error again and again
RuntimeError: Caught RuntimeError in DataLoader worker process 0. Original Traceback (most recent
call last): File "/opt/conda/lib/python3.7/site-packages/torch/utils/data/_utils/worker.py", line
178,
in _worker_loop data = fetcher.fetch(index) File "/opt/conda/lib/python3.7/site-
packages/torch/utils/data/_utils/fetch.py", line 47, in fetch return self.collate_fn(data) File
"/opt/conda/lib/python3.7/site-packages/torch/utils/data/_utils/collate.py", line 79, in
default_collate return [default_collate(samples) for samples in transposed] File
"/opt/conda/lib/python3.7/site-packages/torch/utils/data/_utils/collate.py", line 79, in return
[default_collate(samples) for samples in transposed] File "/opt/conda/lib/python3.7/site-
packages/torch/utils/data/_utils/collate.py", line 55, in default_collate return torch.stack(batch,
0, out=out) RuntimeError: stack expects each tensor to be equal size, but got [47] at entry 0 and
[36] at entry 1
I have tried changing num_workers values,changing batch sizes. I have checked the data and none of the text in it is null, 0 or corrupt in any sense. I have also tried changing max_len in tokenizer but I am not able to find out solution to this problem. Please check and let me know how can I fix it.

data_loader = torch.utils.data.DataLoader( batch_size=batch_size, dataset=data, shuffle=shuffle, num_workers=0, collate_fn=lambda x: x )
Use of Collate_fn in dataloader should be able to solve the problem.