Develop Reference

Create iterator from a Data Frame in Python

I am working on an NLP project using Seq2Seq. I created a data frame from my dataset then created a batch iterator using data loader, see the following code:
# creates lists containing each pair
original_word_pairs = [[w for w in l.split('\t')] for l in lines[:num_examples]]
data = pd.DataFrame(original_word_pairs, columns=["src", "trg"])
# conver the data to tensors and pass to the Dataloader
# to create a batch iterator
class MyData(Dataset):
def __init__(self, X, y):
self.data = X
self.target = y
# TODO: convert this into torch code is possible
self.length = [ np.sum(1 - np.equal(x, 0)) for x in X]
def __getitem__(self, index):
x = self.data[index]
y = self.target[index]
x_len = self.length[index]
return x,y,x_len
def __len__(self):
return len(self.data)
train_dataset = MyData(input_tensor_train, target_tensor_train)
val_dataset = MyData(input_tensor_val, target_tensor_val)
train_dataset = DataLoader(train_dataset, batch_size = BATCH_SIZE,
drop_last=True,
shuffle=True)
test_dataset= DataLoader(val_dataset, batch_size = BATCH_SIZE,
drop_last=True,
shuffle=True)
That is a part of my code, the thing is I want to use the iterators like this
for i, batch in enumerate(iterator):
src = batch.src
trg = batch.trg
But I got an error "AttributeError: 'list' object has no attribute 'src'"
How can I use the iterator and access a specific column?

You can redefine __getitem__ in your Dataset to return a dictionary:
def __getitem__(self, index):
x = self.data[index]
y = self.target[index]
x_len = self.length[index]
return {"src": x, "trg": y, "x_len": x_len}
The default collate_fn of DataLoader will take care to provide a dictionary containing batches instead of single observations, but you need to convert x_len to a tensor into __getitem__ to make it work (or you can pass a custom collate_fn).

AttributeError: 'DataGenerator' object has no attribute 'shape' when I use data generators with Keras

When I use datagenerator below, learned from https://stanford.edu/~shervine/blog/keras-how-to-generate-data-on-the-fly
class DataGenerator(keras.utils.Sequence):
def __init__(self, file_list,batch_size):
"""Constructor can be expanded,
with batch size, dimentation etc.
"""
self.file_list = file_list
self.batch_size = batch_size
self.on_epoch_end()
def __len__(self):
'Take all batches in each iteration'
return int(np.floor(len(self.file_list) / self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
# Generate indexes of the batch
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
# Find list of IDs
list_IDs_temp = [self.list_IDs[k] for k in indexes]
# Generate data
X, y = self.__data_generation(list_IDs_temp)
return X, y
def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.file_list))
def __data_generation(self, file_list_temp):
'Generates data containing batch_size samples'
loc = os.path.abspath('.')
# Generate data
for ID in file_list_temp:
x_file_path = os.path.join(loc, "target",ID)
y_file_path = os.path.join(loc, "newlabel",ID)
# Store sample
X = np.load(x_file_path)
# Store class
y = np.load(y_file_path)
return X, y
I call it using:
batch_size=32
training_generator = DataGenerator(train,batch_size)
validation_generator = DataGenerator(val,batch_size)
bce = tf.keras.losses.BinaryCrossentropy(reduction='sum')
EPOCHS =10
model = tf.keras.models.load_model('./training/20220317.h5',compile=False)
model.compile(optimizer='adam',loss = weightedLoss(100),metrics=['accuracy'])
H = model.fit_generator(generator=training_generator,
validation_data=validation_generator,epochs=EPOCHS)
But when I run the script, it keeps saying :
AttributeError: 'DataGenerator' object has no attribute 'shape'
I tried from tensorflow.python.keras.utils.data_utils import Sequence to fix it, but it doesn't work. What should I do to deal with this error?

Problem solved.
I change from keras.utils import Sequenceto from tensorflow.keras.utils import Sequence
and I use model.fit(x=training_generator)
instead of model.fit_generator

Training stuck at Epoch 3 PyTorch

I am training a custom Encoder-Decoder network but the training gets stuck at Epoch 3. Nothing happens for about 2 hours. I will share the Dataset class and the DataLoader object. The version if CUDA and GPU can be seen in the pic below.
Training stuck here:
nvidia-smi output looks like this:
The __getitem__ method of the dataset class looks like this:
def __init__(self,
images_dir,
annots_dir,
train=True,
img_size=(512, 1536),
stride=4,
model='custom',
transforms=None):
"""
:param root: dataset directory
:param filenames: filenames inside the root directory
:param labels: Object Detection Labels
super(CustomDataset).__init__()
self.images_dir = images_dir
self.annots_dir = annots_dir
self.train = train
self.image_size = img_size
self.stride = stride
self.transforms = transforms
self.model = model
# Load the image and annotation files from the dataset
# self.image_files, self.annot_files = self._load_image_and_annot_files()
self.image_files = [os.path.join(self.images_dir, idx) for idx in os.listdir(self.images_dir)]
self.annot_files = [os.path.join(self.annots_dir, idx) for idx in os.listdir(self.annots_dir)]
def __getitem__(self, index):
"""
:param index: index...0 to N
:return: tensor_image and tensor_label
"""
# Image filename from _load_image_files()
# Load Image with _read_matrix() and label
curr_image_filename = self.image_files[index]
curr_annot_filename = self.annot_files[index]
# curr_image_filename = self.image_files[index]
# curr_annot_filename = self.annot_files[index]
np_image = self._read_matrix(raw_img=curr_image_filename)
np_image_normalized = np.squeeze(self._normalize_raw_img(np_image))
# label = self.labels[index]
boxes, classes, depths, tgts = self._load_annotations(curr_annot_filename)
# Normalize bounding boxes: range [0, 1]
targets_normalized = self._normalize_bbox(np_image_normalized, tgts)
# image and the corresponding label should be a tensor
torch_image = torch.from_numpy(np_image).reshape(1, 512, 1536).float() # dtype: torch.float64
torch_boxes = torch.from_numpy(boxes).type(torch.FloatTensor)
torch_depths = torch.from_numpy(depths)
if self.model == 'fasterrcnn':
# For FasterRCNN: As COCO format
area = (torch_boxes[:, 3] - torch_boxes[:, 1]) * (torch_boxes[:, 2] - torch_boxes[:, 0])
iscrowd = torch.zeros((boxes.shape[0],), dtype=torch.int64)
image_id = torch.Tensor([index])
torch_classes = torch.from_numpy(classes)
target = {'boxes': torch_boxes, 'labels': torch_classes.long(),
'area': area, 'iscrowd': iscrowd, 'image_id': image_id}
return torch_image, target
elif self.model == 'custom':
if self.train:
if self.transforms:
try:
tr = self.transforms()
transform_image, transform_boxes, labels = tr.__call__(np_image, tgts, tgts[:, :4], tgts[:, 4:])
transform_targets = np.hstack((np.array(transform_boxes), labels))
gt_tensor = gt_creator(img_size=self.image_size,
stride=self.stride,
num_classes=8,
label_lists=transform_targets)
return torch.from_numpy(transform_image).float(), gt_tensor
except IndexError:
pass
else:
gt_tensor = gt_creator(img_size=self.image_size,
stride=self.stride,
num_classes=8,
label_lists=targets_normalized)
return torch_image, gt_tensor
else:
return torch_image, targets_normalized
And in the train.py script the DataLoader object is:
train_loader = torch.utils.data.DataLoader(dataset=dataset,
shuffle=True,
batch_size=1,
num_workers=0,
collate_fn=detection_collate,
pin_memory=True)
Why does the training get stuck? Is there an issue with the __getitem__ method? Or the DataLoader?
Thank You.

This happens because torch doesnt restart your dataset, if your data runs out it stops and waits for more input so cycling has to be done manually.
I used something along the lines of
from itertools import cycle
class Dataloader():
#init and whatever
self.__iter__():
return cycle(get_sample()) # get_sample is your current getitem

Dataloader of different sequence of images in a single folder

I'm doing frame generation. For each images in Dataset/train/ folder (e.g 1.png ) I generated a sequence of 100 images and save all of them into a single Dataset/frames/train/ as (1_1.png...1_100.png), here is an example of my folder structure:
Dataset:
train:
1.png
2.png
3.png
.
.
N.png
frames:
train:
1_1.png
1_2.png
.
.
N_100.png
2_1.png
2_2.png
.
.
N_100.png
I have created my custom dataloader where I stack the frames generated as channels to form a sequence, but my problem I don't want to frames from image 2 to overlap with frames from 1 when create a sequence, How can I ensure the different frames don't overlap?
Here is my custom dataloader:
class LevelSetDataset(Dataset):
"""
Dataset object for CNN models
Temporal is defined implicitly
as the number of channels
example:
- X dimension
[H, W, C=number_of_timestap(t)]
- Y dimension
[W, W, C =(t+1)]
"""
def __init__(self, input_image_path:str,
target_image_path:str,
threshold:float=0.5,
num_input_steps:int=3,
num_future_steps:int=1,
image_dimension:int=32,
data_transformations=None,
istraining_mode:bool=True
):
self.input_image_path = input_image_path
self.target_image_path = target_image_path
self.threshold = threshold
self.num_input_steps = num_input_steps
self.num_future_steps = num_future_steps
self.image_dimension = image_dimension
self.data_transformations= data_transformations
self.istraining_mode = istraining_mode
# get a list of input filenames as sort them (e.g. 1.png, 2.png,..,N.png)
input_image_fp = sorted(glob(os.path.join(self.input_image_path , "*")),
key=lambda x: int(os.path.basename(x).split('.')[0])
)
# repeat the input image untill it matches the number of segmentation
# step of the target image
self.input_image_fp = [i for i in input_image_fp for _ in range(100)]
# get a list of the target filenames and sort them by the first id and second
# id after the underscore (e.g. 1_1.png, 1_2,..,N_M.png)
self.target_image_fp= sorted(glob(os.path.join(self.target_image_path , "*")),
key=lambda x: (int(os.path.basename(x).split('_')[0]),
int(os.path.basename(x).split('_')[1].split('.')[0]))
)
# check if in training mode
# to apply transformations
if (self.data_transformations is None) and (self.istraining_mode):
self.data_transformations= torchvision.transforms.Compose([
torchvision.transforms.Resize(size=(self.image_dimension,self.image_dimension),
interpolation=Image.BILINEAR),
torchvision.transforms.RandomHorizontalFlip(p=0.5),
torchvision.transforms.RandomVerticalFlip(p=0.5),
torchvision.transforms.ToTensor()
])
if (self.data_transformations is None) and (not self.istraining_mode):
self.data_transformations== torchvision.transforms.Compose([
torchvision.transforms.Resize(size=(self.image_dimension,self.image_dimension),
interpolation=Image.BILINEAR),
torchvision.transforms.ToTensor()
])
self.transforms = self.data_transformations
def _create_binary_mask(self, x):
x[x>=self.threshold] = 1
x[x <self.threshold] = 0
return x
def _stat_norm(self, x):
norm =torchvision.transforms.Compose([torchvision.transforms.Resize(
size=(self.image_dimension,self.image_dimension),
interpolation=Image.BILINEAR),
torchvision.transforms.ToTensor()])
return norm(x)
def __len__(self):
return len(self.target_image_fp) - (self.num_input_steps+self.num_future_steps)
def __getitem__(self, index):
X = torch.zeros((self.image_dimension, self.image_dimension, self.num_input_steps+1))
for step_idx, step in enumerate(np.arange(index, self.num_input_steps, 1)):
target_image = Image.open(self.target_image_fp[step+self.num_input_steps+self.num_future_steps-1])
target_image = self.transforms(target_image)
target_image = self._create_binary_mask(target_image)
X[:, :, step_idx] = target_image # (t+1)
input_img = Image.open(self.input_image_fp[index]).convert('L')
# input_img = self.transforms(input_img)
input_img = self.transforms(input_img)
X[:, :, 0] = input_img
target_image = Image.open(self.target_image_fp[index+self.num_input_steps+self.num_future_steps-1])
target_image = self.transforms(target_image)
target_image = self._create_binary_mask(target_image)
image_name = self.target_image_fp[index+self.num_input_steps+self.num_future_steps-1].split('/')[-1]
Y = target_image
return X, Y, image_name

Tensorflow: predicting a point from an image, training model with a point labels

I want to create a model which can predict a point from an image.
I have a dataset with training images. Those images are splitted between 24 dirs.
I have prepared an json file containing a (x, y) values for every image.
example:
"dir22": {
"frame_00001_rgb": {
"x": 363.693829827852,
"y": 278.2191728859505
},
"frame_00002_rgb": {
"x": 330.9709780765119,
"y": 283.34142472069004
},
...
...
"dir23": {
"frame_00001_rgb": {
"x": 212.5232358000000,
"y": 156.3342191728855
},
"frame_00002_rgb": {
"x": 230.69497097807351,
"y": 253.75341424720690
},
My model looks like this:
img_width, img_height = 640, 480
train_data_dir = 'v_data/train'
epochs = 10
batch_size = 16
input_tensor = tf.keras.Input(shape=(img_width,img_height,3))
base_model = tf.keras.applications.ResNet50(weights='imagenet',include_top=False ,input_tensor=input_tensor)
top_model = tf.keras.Sequential()
top_model.add(tf.keras.Flatten(input_shape=base_model.output_shape[1:]))
top_model.add(tf.keras.Dense(128, activation='relu'))
top_model.add(tf.keras.Dense(128, activation='relu'))
top_model.add(tf.keras.Dense(2))
model = tf.keras.Model(input= base_model.input, output= top_model(base_model.output))
for layer in model.layers[-15:]:
layer.trainable = False
optimizer = tf.keras.optimizers.RMSprop(0.001)
model.compile(loss='mse',
optimizer=optimizer,
metrics=['mae', 'mse'])
now I have loaded images from my dir:
train_datagen = tf.keras.preprocessing.image.ImageDataGenerator()
train_generator = train_datagen.flow_from_directory(
train_data_dir,
target_size=(img_height, img_width),
batch_size=batch_size)
Found 15678 images belonging to 24 classes.
now how can I assign a label for each image and train my model with it?

For this, you need to write a custom data generator.
Importing necessary libraries
import os
import pandas as pd
from skimage.io import imread # Used for image processing
from skimage.transform import resize # Used for image processing
import json
import numpy as np
Defining our own data generator
I followed this link to get an idea of how to do this. And customized it to your problem.
We need to fill in the following functions.
class DataGenerator(tf.keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, directory, target_json, batch_size=32, target_size=(128, 128), shuffle=True):
...
def __len__(self):
'Denotes the number of batches per epoch'
...
def __getitem__(self, index):
'Generate one batch of data'
...
def on_epoch_end(self):
'Updates indexes after each epoch'
...
def __data_generation(self, list_paths, list_paths_wo_ext):
'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)
...
Let's look at what variables we define
self.target_size = # Final size of the images
self.batch_size = # Batch size
self.target_json = # Path to the json file
self.directory = # Where the training data is
self.img_paths = # Contains image paths with extension
self.img_paths_wo_ext = # Contains the image paths without extension
self.targets = # The dataframe containing targets loaded from the json
self.shuffle = # Shuffle data at start of each epoch?
The JSON file
Your JSON file needs to be exactly in this format. This is probably what you have exactly too. But make sure it's 100% this format.
{'dir20': {'frame_00001_rgb': {'x': 363.693829827852, 'y': 278.2191728859505}, 'frame_00002_rgb': {'x': 330.9709780765119, 'y': 283.34142472069004}}, 'dir21': {'frame_00001_rgb': {'x': 363.693829827852, 'y': 278.2191728859505}, 'frame_00002_rgb': {'x': 330.9709780765119, 'y': 283.34142472069004}}, 'dir22': {'frame_00001_rgb': {'x': 363.693829827852, 'y': 278.2191728859505}, 'frame_00002_rgb': {'x': 330.9709780765119, 'y': 283.34142472069004}}, 'dir23': {'frame_00001_rgb': {'x': 363.693829827852, 'y': 278.2191728859505}, 'frame_00002_rgb': {'x': 330.9709780765119, 'y': 283.34142472069004}}, 'dir24': {'frame_00001_rgb': {'x': 212.5232358, 'y': 156.3342191728855}, 'frame_00002_rgb': {'x': 230.6949709780735, 'y': 253.7534142472069}}}
Next we need to convert this to a pandas dataframe. For that we define the following function. It's a bit complex due to the nature of your file. But here's what's happening.
Load the json and create a dataframe which contain columns like dir20.frame_00002_rgb.x.
Create a multi index by splitting the column to 3 levels (e.g. dir20, frame_00002, x)
Use stack to bring both dir* and frame_* as indices
Reformat the index so that it contains the full path for each image and each record has two columns (x and y).
def json_to_df(json_path, directory):
with open(json_path,'r') as f:
s = json.load(f)
df = pd.io.json.json_normalize(s)
ind = pd.MultiIndex.from_tuples([col.split('.') for col in df.columns])
df.columns = ind
df = df.stack(level=[0,1])
df = df.set_index(df.index.droplevel(0))
df = df.set_index(pd.Index([os.path.sep.join([directory]+list(c)) for c in df.index.values]))
return df
Rest of the code
I won't go in to great details of what's going on in the other parts as it is quite straight forward. But we're essentially getting a single batch of data by reading the images, resizing and getting the correct x, y values from the dataframe we generated.
Full code
Here's the full code for the data generator.
class DataGenerator(tf.keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, directory, target_json, batch_size=32, target_size=(128, 128), shuffle=True):
'Initialization'
self.target_size = target_size
self.batch_size = batch_size
self.target_json = target_json
self.directory = directory
self.img_paths = []
self.img_paths_wo_ext = []
for root, dirs, files in os.walk(directory):
for file in files:
if file.lower().endswith(".jpg") or file.lower().endswith(".png"):
self.img_paths.append(os.path.join(root, file))
self.img_paths_wo_ext.append(os.path.splitext(os.path.join(root, file))[0])
def json_to_df(json_path, directory):
with open(json_path,'r') as f:
s = json.load(f)
df = pd.io.json.json_normalize(s)
ind = pd.MultiIndex.from_tuples([col.split('.') for col in df.columns])
df.columns = ind
df = df.stack(level=[0,1])
df = df.set_index(df.index.droplevel(0))
df = df.set_index(pd.Index([os.path.sep.join([directory]+list(c)) for c in df.index.values]))
return df
self.targets = json_to_df(self.target_json, self.directory)
self.shuffle = shuffle
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.img_paths) / self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
# Generate indexes of the batch
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
# Find list of IDs
list_paths = [self.img_paths[k] for k in indexes]
list_paths_wo_ext = [self.img_paths_wo_ext[k] for k in indexes]
# Generate data
X, y = self.__data_generation(list_paths, list_paths_wo_ext)
return X, y
def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.img_paths))
if self.shuffle == True:
np.random.shuffle(self.indexes)
def __data_generation(self, list_paths, list_paths_wo_ext):
'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)
# Initialization
X = np.empty((self.batch_size, *self.target_size, 3))
y = self.targets.loc[list_paths_wo_ext].values
# Generate data
for i, ID in enumerate(list_paths):
# Store sample
X[i,] = resize(imread(ID),self.target_size)
return X, y
Using the datagenerator
Here's how you'd use the data generator.
train_datagen = iter(DataGenerator(train_data_dir, './train/data.json', batch_size=2))
x, y = next(train_datagen)
print(x)
print(y)
which gives,
[[0.01377145 0.01377145 0.01377145]
[0.00242393 0.00242393 0.00242393]
[0. 0. 0. ]
...
[0.0037837 0.0037837 0.0037837 ]
[0.0037837 0.0037837 0.0037837 ]
[0.0037837 0.0037837 0.0037837 ]]
...
[[0.37398897 0.3372549 0.17647059]
[0.38967525 0.35294118 0.19215686]
[0.42889093 0.39215686 0.23137255]
...
[0.72156863 0.62889093 0.33085172]
[0.71372549 0.61176471 0.31764706]
[0.70588235 0.59359681 0.30340074]]]]
[[363.69382983 278.21917289]
[330.97097808 283.34142472]]