Develop Reference

Getting error while extracting key value pair using LayoutLMV2 model

I am trying to extract key value pair from scanned invoices document using LayoutLMV2 model but I am getting error. Installation guide. I am just trying to check how the model is predicting the key value pair from the document or do I need to fine tune the model in own data set. Need help in this.
from transformers import PreTrainedTokenizerBase, LayoutLMv2FeatureExtractor, LayoutLMv2Processor, AutoTokenizer, LayoutLMv2ForRelationExtraction, AdamW
from transformers.file_utils import PaddingStrategy
import torch
from torch import nn
from dataclasses import dataclass
from typing import Dict, Tuple, Optional, Union
from datasets import load_dataset
from torch.utils.data import DataLoader
from torchvision.transforms import ToPILImage
from torchvision import transforms
from PIL import Image, ImageFont, ImageDraw, ImageEnhance, ImageFilter
import numpy as np
import pytesseract
pytesseract.pytesseract.tesseract_cmd = r'C:\Users\name\AppData\Local\Programs\Tesseract-OCR\tesseract.exe'
feature_extractor = LayoutLMv2FeatureExtractor(apply_ocr=True)
tokenizer = AutoTokenizer.from_pretrained(path_1, pad_token='<pad>')
processor = LayoutLMv2Processor(feature_extractor, tokenizer)
model = LayoutLMv2ForRelationExtraction.from_pretrained(path_1)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(device)
image_file = 'image2.png'
image = Image.open(image_file).convert('RGB')
image.size
encoded_inputs = processor(image, return_tensors="pt")
encoded_inputs.keys()
for k,v in encoded_inputs.items():
print(k, v.shape)
for k,v in encoded_inputs.items():
encoded_inputs[k] = v.to(model.device)
# forward pass
outputs = model(**encoded_inputs)
This is the error I am getting
TypeError Traceback (most recent call last)
c:\Users\name\Parallel Project\Trans_LayoutXLM.ipynb Cell 7 in <cell line: 5>()
2 encoded_inputs[k] = v.to(model.device)
4 # forward pass
----> 5 outputs = model(**encoded_inputs)
File c:\Users\name\.conda\envs\layoutlmft\lib\site-packages\torch\nn\modules\module.py:1130, in Module._call_impl(self, *input, **kwargs)
1126 # If we don't have any hooks, we want to skip the rest of the logic in
1127 # this function, and just call forward.
1128 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1129 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1130 return forward_call(*input, **kwargs)
1131 # Do not call functions when jit is used
1132 full_backward_hooks, non_full_backward_hooks = [], []
File c:\Users\name\.conda\envs\layoutlmft\lib\site-packages\transformers\models\layoutlmv2\modeling_layoutlmv2.py:1598, in LayoutLMv2ForRelationExtraction.forward(self, input_ids, bbox, labels, image, attention_mask, token_type_ids, position_ids, head_mask, entities, relations)
1596 sequence_output, image_output = outputs[0][:, :seq_length], outputs[0][:, seq_length:]
1597 sequence_output = self.dropout(sequence_output)
-> 1598 loss, pred_relations = self.extractor(sequence_output, entities, relations)
1600 return RegionExtractionOutput(
1601 loss=loss,
1602 entities=entities,
(...)
1605 hidden_states=outputs[0],
1606 )
File c:\Users\name\.conda\envs\layoutlmft\lib\site-packages\torch\nn\modules\module.py:1130, in Module._call_impl(self, *input, **kwargs)
1126 # If we don't have any hooks, we want to skip the rest of the logic in
1127 # this function, and just call forward.
1128 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1129 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1130 return forward_call(*input, **kwargs)
1131 # Do not call functions when jit is used
1132 full_backward_hooks, non_full_backward_hooks = [], []
...
-> 1421 batch_size = len(relations)
1422 new_relations = []
1423 for b in range(batch_size):
TypeError: object of type 'NoneType' has no len()

import numpy as np
from transformers import LayoutLMv2Processor, LayoutLMv2ForTokenClassification
from datasets import load_dataset
from PIL import Image, ImageDraw, ImageFont
processor = LayoutLMv2Processor.from_pretrained("microsoft/layoutlmv2-base-uncased")
model = LayoutLMv2ForTokenClassification.from_pretrained("nielsr/layoutlmv2-finetuned-funsd")
dataset = load_dataset("nielsr/funsd", split="test")
labels = dataset.features['ner_tags'].feature.names
id2label = {v: k for v, k in enumerate(labels)}
label2color = {'question':'blue', 'answer':'green', 'header':'orange', 'other':'red'}
def unnormalize_box(bbox, width, height):
return [
width * (bbox[0] / 1000),
height * (bbox[1] / 1000),
width * (bbox[2] / 1000),
height * (bbox[3] / 1000),
]
def iob_to_label(label):
label = label[2:]
if not label:
return 'other'
return label
image_path="invoice.jpg"
# load image example
image = Image.open(image_path).convert("RGB")
image
def process_image(image):
width, height = image.size
# encode
encoding = processor(image, truncation=True, return_offsets_mapping=True, return_tensors="pt")
offset_mapping = encoding.pop('offset_mapping')
# forward pass
outputs = model(**encoding)
# get predictions
predictions = outputs.logits.argmax(-1).squeeze().tolist()
token_boxes = encoding.bbox.squeeze().tolist()
# only keep non-subword predictions
is_subword = np.array(offset_mapping.squeeze().tolist())[:,0] != 0
true_predictions = [id2label[pred] for idx, pred in enumerate(predictions) if not is_subword[idx]]
true_boxes = [unnormalize_box(box, width, height) for idx, box in enumerate(token_boxes) if not is_subword[idx]]
# draw predictions over the image
draw = ImageDraw.Draw(image)
font = ImageFont.load_default()
for prediction, box in zip(true_predictions, true_boxes):
predicted_label = iob_to_label(prediction).lower()
draw.rectangle(box, outline=label2color[predicted_label])
draw.text((box[0]+10, box[1]-10), text=predicted_label, fill=label2color[predicted_label], font=font)
return image
process_image(image)

Segformer: IndexError: Target 151 is out of bounds

I've been working on this Segformer (using transformers for image segmentation) I've been having issues with getting a message stating that the target is out of bounds.
So here is the code that I am using.
!pip install --upgrade transformers
!nvidia-smi
from torch.utils.data import Dataset, DataLoader
from transformers import AdamW
import torch
from torch import nn
from sklearn.metrics import accuracy_score
from tqdm.notebook import tqdm
import os
from PIL import Image
from transformers import SegformerForSemanticSegmentation,
SegformerFeatureExtractor
import pandas as pd
import cv2
import numpy as np
import albumentations as aug
os.getcwd()
ok, from here this is my current working directoy:
'/data/users/me/Rugd_dataset'
My images are stored in two folders one called images, the nest mask. Each have a sub directory of test and train
WIDTH = 512
Height = 512
class ImageSegmentationDataset(Dataset):
"""Image segmentation dataset."""
def __init__(self, root_dir, feature_extractor, transforms=None, train=True):
"""
Args:
root_dir (string): Root directory of the dataset containing the images + annotations.
feature_extractor (SegFormerFeatureExtractor): feature extractor to prepare images + segmentation maps.
train (bool): Whether to load "training" or "validation" images + annotations.
"""
self.root_dir = root_dir
self.feature_extractor = feature_extractor
self.train = train
self.transforms = transforms
sub_path = "train" if self.train else "test"
self.img_dir = os.path.join(self.root_dir, "images", sub_path)
self.ann_dir = os.path.join(self.root_dir, "mask", sub_path)
# read images
image_file_names = []
for root, dirs, files in os.walk(self.img_dir):
image_file_names.extend(files)
self.images = sorted(image_file_names)
# read annotations
annotation_file_names = []
for root, dirs, files in os.walk(self.ann_dir):
annotation_file_names.extend(files)
self.annotations = sorted(annotation_file_names)
assert len(self.images) == len(self.annotations), "There must be as many images as there are segmentation maps"
def __len__(self):
return len(self.images)
def __getitem__(self, idx):
image = cv2.imread(os.path.join(self.img_dir,
self.images[idx]))
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
segmentation_map = cv2.imread(os.path.join(self.ann_dir, self.annotations[idx]))
segmentation_map = cv2.cvtColor(segmentation_map, cv2.COLOR_BGR2GRAY)
# image = Image.open()
# segmentation_map = Image.open()
if self.transforms is not None:
augmented = self.transforms(image=image, mask=segmentation_map)
# randomly crop + pad both image and segmentation map to same size
encoded_inputs = self.feature_extractor(augmented['image'], augmented['mask'], return_tensors="pt")
else:
encoded_inputs = self.feature_extractor(image, segmentation_map, return_tensors="pt")
for k,v in encoded_inputs.items():
encoded_inputs[k].squeeze_() # remove batch dimension
return encoded_inputs
transform = aug.Compose([
aug.Flip(p=0.5)
])
dir_path = r'images/train'
count = 0
# Iterate directory
for path in os.listdir(dir_path):
# check if current path is a file
if os.path.isfile(os.path.join(dir_path, path)):
count += 1
print('File count:', count)
current count = File count: 7386
dir_path = r'mask/train'
count = 0
# Iterate directory
for path in os.listdir(dir_path):
# check if current path is a file
if os.path.isfile(os.path.join(dir_path, path)):
count += 1
print('File count:', count)
this is what I receive as a File count: 7386. So my paths to the images seem to fit.
root_dir = '/data/users/christopher.kehl/Rugd_dataset'
feature_extractor = SegformerFeatureExtractor(align=False, reduce_zero_label=False)
train_dataset = ImageSegmentationDataset(root_dir=root_dir, feature_extractor=feature_extractor, transforms=transform)
valid_dataset = ImageSegmentationDataset(root_dir=root_dir, feature_extractor=feature_extractor, transforms=None, train=False)
print("Number of training examples:", len(train_dataset))
print("Number of validation examples:", len(valid_dataset))
Number of training examples: 7387
Number of validation examples: 100
encoded_inputs = train_dataset[-1]
so the index here is supposed to be zero, but I can only get this to work with a 1 or an -1, anything other than 0.
encoded_inputs["pixel_values"].shape
torch.Size([3, 512, 512])
encoded_inputs["labels"].shape
torch.Size([512, 512])
encoded_inputs["labels"].shape
torch.Size([512, 512])
encoded_inputs["labels"].squeeze().unique()
tensor([ 29, 60, 76, 91, 107])
mask = encoded_inputs["labels"].numpy()
import matplotlib.pyplot as plt
plt.imshow(mask)
shows the images of a mask
from torch.utils.data import DataLoader
train_dataloader = DataLoader(train_dataset, batch_size=4, shuffle=True)
valid_dataloader = DataLoader(valid_dataset, batch_size=4)
batch = next(iter(train_dataloader))
for k,v in batch.items():
print(k, v.shape)
pixel_values torch.Size([4, 3, 512, 512])
labels torch.Size([4, 512, 512])
batch["labels"].shape
torch.Size([4, 512, 512])
classes = pd.read_csv('class_dict_seg.csv')['name']
id2label = classes.to_dict()
label2id = {v: k for k, v in id2label.items()}
model = SegformerForSemanticSegmentation.from_pretrained("nvidia/mit-b5", ignore_mismatched_sizes=True,
num_labels=len(id2label), id2label=id2label, label2id=label2id,
reshape_last_stage=True)
optimizer = AdamW(model.parameters(), lr=0.00006)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(device)
print("Model Initialized!")
for epoch in range(1, 11): # loop over the dataset multiple times
print("Epoch:", epoch)
pbar = tqdm(train_dataloader)
accuracies = []
losses = []
val_accuracies = []
val_losses = []
model.train()
for idx, batch in enumerate(pbar):
# get the inputs;
pixel_values = batch["pixel_values"].to(device)
labels = batch["labels"].to(device)
# zero the parameter gradients
optimizer.zero_grad()
# forward
outputs = model(pixel_values=pixel_values, labels=labels)
# evaluate
upsampled_logits = nn.functional.interpolate(outputs.logits, size=labels.shape[-2:], mode="bilinear", align_corners=False)
predicted = upsampled_logits.argmax(dim=1)
mask = (labels != 255) # we don't include the background class in the accuracy calculation
pred_labels = predicted[mask].detach().cpu().numpy()
true_labels = labels[mask].detach().cpu().numpy()
accuracy = accuracy_score(pred_labels, true_labels)
loss = outputs.loss
accuracies.append(accuracy)
losses.append(loss.item())
pbar.set_postfix({'Batch': idx, 'Pixel-wise accuracy': sum(accuracies)/len(accuracies), 'Loss': sum(losses)/len(losses)})
# backward + optimize
loss.backward()
optimizer.step()
else:
model.eval()
with torch.no_grad():
for idx, batch in enumerate(valid_dataloader):
pixel_values = batch["pixel_values"].to(device)
labels = batch["labels"].to(device)
outputs = model(pixel_values=pixel_values, labels=labels)
upsampled_logits = nn.functional.interpolate(outputs.logits, size=labels.shape[-2:], mode="bilinear", align_corners=False)
predicted = upsampled_logits.argmax(dim=1)
mask = (labels != 255) # we don't include the background class in the accuracy calculation
pred_labels = predicted[mask].detach().cpu().numpy()
true_labels = labels[mask].detach().cpu().numpy()
accuracy = accuracy_score(pred_labels, true_labels)
val_loss = outputs.loss
val_accuracies.append(accuracy)
val_losses.append(val_loss.item())
print(f"Train Pixel-wise accuracy: {sum(accuracies)/len(accuracies)}\
Train Loss: {sum(losses)/len(losses)}\
Val Pixel-wise accuracy: {sum(val_accuracies)/len(val_accuracies)}\
Val Loss: {sum(val_losses)/len(val_losses)}")
Here is the error Block:
IndexError: Target 151 is out of bounds.
Full Trace
---------------------------------------------------------------------------
IndexError Traceback (most recent call last)
<ipython-input-32-50d0deee01ff> in <module>
16
17 # forward
---> 18 outputs = model(pixel_values=pixel_values, labels=labels)
19
20 # evaluate
~/.local/lib/python3.6/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1101 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102 return forward_call(*input, **kwargs)
1103 # Do not call functions when jit is used
1104 full_backward_hooks, non_full_backward_hooks = [], []
~/.local/lib/python3.6/site-packages/transformers/models/segformer/modeling_segformer.py in forward(self, pixel_values, labels, output_attentions, output_hidden_states, return_dict)
807 )
808 loss_fct = CrossEntropyLoss(ignore_index=self.config.semantic_loss_ignore_index)
--> 809 loss = loss_fct(upsampled_logits, labels)
810
811 if not return_dict:
~/.local/lib/python3.6/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1101 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102 return forward_call(*input, **kwargs)
1103 # Do not call functions when jit is used
1104 full_backward_hooks, non_full_backward_hooks = [], []
~/.local/lib/python3.6/site-packages/torch/nn/modules/loss.py in forward(self, input, target)
1150 return F.cross_entropy(input, target, weight=self.weight,
1151 ignore_index=self.ignore_index, reduction=self.reduction,
-> 1152 label_smoothing=self.label_smoothing)
1153
1154
~/.local/lib/python3.6/site-packages/torch/nn/functional.py in cross_entropy(input, target, weight, size_average, ignore_index, reduce, reduction, label_smoothing)
2844 if size_average is not None or reduce is not None:
2845 reduction = _Reduction.legacy_get_string(size_average, reduce)
-> 2846 return torch._C._nn.cross_entropy_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index, label_smoothing)
2847
2848
IndexError: Target 150 is out of bounds.

Input tensors to a Functional must come from `tf.keras.Input`. Received: 0 (missing previous layer metadata) and cant find the cause

I got an error of ValueError: Input tensors to a Functional must come from tf.keras.Input. Received: 0 (missing previous layer metadata) and i cant find the cause
this is my error trace and my code
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-15-8058f3a2fd50> in <module>()
6 test_loss, test_accuracy = eg.test(dg.user_test)
7 print('Test set: Loss=%.4f ; Accuracy=%.1f%%' % (test_loss, test_accuracy * 100))
----> 8 eg.save_embeddings('embeddings.csv')
7 frames
<ipython-input-5-54ff9897b1c3> in save_embeddings(self, file_name)
66 inp = self.m.input # input placeholder
67 outputs = [layer.output for layer in self.m.layers] # all layer outputs
---> 68 functor = K.function([inp, K.learning_phase()], outputs ) # evaluation function
69
70 #append embeddings to vectors
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/backend.py in function(inputs, outputs, updates, name, **kwargs)
3934 from tensorflow.python.keras import models # pylint: disable=g-import-not-at-top
3935 from tensorflow.python.keras.utils import tf_utils # pylint: disable=g-import-not-at-top
-> 3936 model = models.Model(inputs=inputs, outputs=outputs)
3937
3938 wrap_outputs = isinstance(outputs, list) and len(outputs) == 1
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py in __new__(cls, *args, **kwargs)
240 # Functional model
241 from tensorflow.python.keras.engine import functional # pylint: disable=g-import-not-at-top
--> 242 return functional.Functional(*args, **kwargs)
243 else:
244 return super(Model, cls).__new__(cls, *args, **kwargs)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/tracking/base.py in _method_wrapper(self, *args, **kwargs)
455 self._self_setattr_tracking = False # pylint: disable=protected-access
456 try:
--> 457 result = method(self, *args, **kwargs)
458 finally:
459 self._self_setattr_tracking = previous_value # pylint: disable=protected-access
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/functional.py in __init__(self, inputs, outputs, name, trainable)
113 # 'arguments during initialization. Got an unexpected argument:')
114 super(Functional, self).__init__(name=name, trainable=trainable)
--> 115 self._init_graph_network(inputs, outputs)
116
117 #trackable.no_automatic_dependency_tracking
/usr/local/lib/python3.6/dist-packages/tensorflow/python/training/tracking/base.py in _method_wrapper(self, *args, **kwargs)
455 self._self_setattr_tracking = False # pylint: disable=protected-access
456 try:
--> 457 result = method(self, *args, **kwargs)
458 finally:
459 self._self_setattr_tracking = previous_value # pylint: disable=protected-access
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/functional.py in _init_graph_network(self, inputs, outputs)
142 base_layer_utils.create_keras_history(self._nested_outputs)
143
--> 144 self._validate_graph_inputs_and_outputs()
145
146 # A Network does not create weights of its own, thus it is already
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/functional.py in _validate_graph_inputs_and_outputs(self)
637 'must come from `tf.keras.Input`. '
638 'Received: ' + str(x) +
--> 639 ' (missing previous layer metadata).')
640 # Check that x is an input tensor.
641 # pylint: disable=protected-access
ValueError: Input tensors to a Functional must come from `tf.keras.Input`. Received: 0 (missing previous layer metadata).
and this is my snipped code:
class EmbeddingsGenerator:
def __init__(self, train_users, data):
self.train_users = train_users
#preprocess
self.data = data.sort_values(by=['timestamp'])
#make them start at 0
self.data['userId'] = self.data['userId'] - 1
self.data['itemId'] = self.data['itemId'] - 1
self.user_count = self.data['userId'].max() + 1
self.movie_count = self.data['itemId'].max() + 1
self.user_movies = {} #list of rated movies by each user
for userId in range(self.user_count):
self.user_movies[userId] = self.data[self.data.userId == userId]['itemId'].tolist()
self.m = self.model()
def model(self, hidden_layer_size=100):
m = Sequential()
m.add(Dense(hidden_layer_size, input_shape=(1, self.movie_count)))
m.add(Dropout(0.2))
m.add(Dense(self.movie_count, activation='softmax'))
m.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
return m
def generate_input(self, user_id):
'''
Returns a context and a target for the user_id
context: user's history with one random movie removed
target: id of random removed movie
'''
user_movies_count = len(self.user_movies[user_id])
#picking random movie
random_index = np.random.randint(0, user_movies_count-1) # -1 avoids taking the last movie
#setting target
target = np.zeros((1, self.movie_count))
target[0][self.user_movies[user_id][random_index]] = 1
#setting context
context = np.zeros((1, self.movie_count))
context[0][self.user_movies[user_id][:random_index] + self.user_movies[user_id][random_index+1:]] = 1
return context, target
def train(self, nb_epochs = 300, batch_size = 10000):
'''
Trains the model from train_users's history
'''
for i in range(nb_epochs):
print('%d/%d' % (i+1, nb_epochs))
batch = [self.generate_input(user_id=np.random.choice(self.train_users) - 1) for _ in range(batch_size)]
X_train = np.array([b[0] for b in batch])
y_train = np.array([b[1] for b in batch])
self.m.fit(X_train, y_train, epochs=1, validation_split=0.5)
def test(self, test_users, batch_size = 100000):
'''
Returns [loss, accuracy] on the test set
'''
batch_test = [self.generate_input(user_id=np.random.choice(test_users) - 1) for _ in range(batch_size)]
X_test = np.array([b[0] for b in batch_test])
y_test = np.array([b[1] for b in batch_test])
return self.m.evaluate(X_test, y_test)
def save_embeddings(self, file_name):
'''
Generates a csv file containg the vector embedding for each movie.
'''
inp = self.m.input # input placeholder
outputs = [layer.output for layer in self.m.layers] # all layer outputs
functor = K.function([inp, K.learning_phase()], outputs ) # evaluation function
#append embeddings to vectors
vectors = []
for movie_id in range(self.movie_count):
movie = np.zeros((1, 1, self.movie_count))
movie[0][0][movie_id] = 1
layer_outs = functor([movie])
vector = [str(v) for v in layer_outs[0][0][0]]
vector = '|'.join(vector)
vectors.append([movie_id, vector])
#saves as a csv file
embeddings = pd.DataFrame(vectors, columns=['item_id', 'vectors']).astype({'item_id': 'int32'})
embeddings.to_csv(file_name, sep=';', index=False)
files.download(file_name)
this is the part of code which call the save_embeddings method
if True: # Generate embeddings?
eg = EmbeddingsGenerator(dg.user_train, pd.read_csv('ml-100k/u.data', sep='\t', names=['userId', 'itemId', 'rating', 'timestamp']))
eg.train(nb_epochs=300)
train_loss, train_accuracy = eg.test(dg.user_train)
print('Train set: Loss=%.4f ; Accuracy=%.1f%%' % (train_loss, train_accuracy * 100))
test_loss, test_accuracy = eg.test(dg.user_test)
print('Test set: Loss=%.4f ; Accuracy=%.1f%%' % (test_loss, test_accuracy * 100))
eg.save_embeddings('embeddings.csv')

Pytorch double image input : broken pipe with dataloader

I am developing a neural network in python with pytorch in order to classify a dataset of pairs of images. So I wanted to return as an output the 2 images and the ground truth but whenever I try to use the data loader I get the error "Broken pipe".
I want it to work as a classifier so I'm using this link (pytorch classifier CIFAR10) as an example
Here is my code :
###Defining class
class continuousImgDataset(Dataset):
def __init__(self, tabDataset, transform=None):
"""
Args:
tabDataset : Contains the imported data like : Item 1 = [im1,im2,ground-truth]
transform (callable, optional): Optional transform to be applied
on a sample.
"""
self.tabPairImg = tabDataset
self.transform = transform
def __len__(self):
return len(self.tabPairImg)
def __getitem__(self, idx):
img1 = self.tabPairImg[idx][0]
img2 = self.tabPairImg[idx][1]
label = self.tabPairImg[idx][2]
if self.transform:
img1 = self.transform(img1)
img2 = self.transform(img2)
return img1, img2,label
transform = transforms.Compose(
[transforms.Scale((32,32)),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
###Make the training and testing dataset
custom_dataset = continuousImgDataset(pairImg,transform)
train_dataset = continuousImgDataset(pairImg[: int(len(custom_dataset) * .90)],transform)
test_dataset = continuousImgDataset(pairImg[int(len(custom_dataset) * .90) : int(len(custom_dataset))],transform)
print(len(custom_dataset.tabPairImg)) #Output :22620
print(len(train_dataset.tabPairImg)) #Output : 20358
print(len(test_dataset.tabPairImg)) #Output : 2262
### Loaders
trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=4,
shuffle=True, num_workers=2)
testloader = torch.utils.data.DataLoader(test_dataset, batch_size=4,
shuffle=False, num_workers=2)
classes = ('joint','disjoint')
### Image show and Error
"""From the official website but a little modified to have 2 imgs"""
def imshow(img):
img = img / 2 + 0.5 # unnormalize
npimg = img.numpy()
plt.imshow(np.transpose(npimg, (1, 2, 0)))
plt.show()
# get some random training images
dataiter = iter(trainloader) #ERROR HERE : Broken pipe
images1,images2, labels = dataiter.next()
# show images
imshow(torchvision.utils.make_grid(images1))
imshow(torchvision.utils.make_grid(images2))
# print labels
print(' '.join('%5s' % classes[labels[j]] for j in range(2)))
I may not have fully understand what the dataloader was expecting in order to iterate on it. Any help would be appreciated :)
EDIT : Here is the full error :
BrokenPipeError Traceback (most recent call last)
<ipython-input-58-314e85cc8fbb> in <module>
7
8 # get some random training images
----> 9 dataiter = iter(trainloader)
10 images1,images2, labels = dataiter.next()
11
C:\ProgramData\Anaconda3\lib\site-packages\torch\utils\data\dataloader.py in __iter__(self)
191
192 def __iter__(self):
--> 193 return _DataLoaderIter(self)
194
195 def __len__(self):
C:\ProgramData\Anaconda3\lib\site-packages\torch\utils\data\dataloader.py in __init__(self, loader)
467 # before it starts, and __del__ tries to join but will get:
468 # AssertionError: can only join a started process.
--> 469 w.start()
470 self.index_queues.append(index_queue)
471 self.workers.append(w)
C:\ProgramData\Anaconda3\lib\multiprocessing\process.py in start(self)
110 'daemonic processes are not allowed to have children'
111 _cleanup()
--> 112 self._popen = self._Popen(self)
113 self._sentinel = self._popen.sentinel
114 # Avoid a refcycle if the target function holds an indirect
C:\ProgramData\Anaconda3\lib\multiprocessing\context.py in _Popen(process_obj)
221 #staticmethod
222 def _Popen(process_obj):
--> 223 return _default_context.get_context().Process._Popen(process_obj)
224
225 class DefaultContext(BaseContext):
C:\ProgramData\Anaconda3\lib\multiprocessing\context.py in _Popen(process_obj)
320 def _Popen(process_obj):
321 from .popen_spawn_win32 import Popen
--> 322 return Popen(process_obj)
323
324 class SpawnContext(BaseContext):
C:\ProgramData\Anaconda3\lib\multiprocessing\popen_spawn_win32.py in __init__(self, process_obj)
87 try:
88 reduction.dump(prep_data, to_child)
---> 89 reduction.dump(process_obj, to_child)
90 finally:
91 set_spawning_popen(None)
C:\ProgramData\Anaconda3\lib\multiprocessing\reduction.py in dump(obj, file, protocol)
58 def dump(obj, file, protocol=None):
59 '''Replacement for pickle.dump() using ForkingPickler.'''
---> 60 ForkingPickler(file, protocol).dump(obj)
61
62 #
BrokenPipeError: [Errno 32] Broken pipe

Tensorflow Custom Estimator: How to implement a `input_fn` function that returns a list of labels, and a list of features ?

I am trying to convert my Tensorflow graph to use a custom tensorflow estimator, but I am getting stuck defining the function for input_fn ; I am currently getting an error.
This is the function I use to generate my input data and labels
data_index = 0
epoch_index = 0
recEpoch_indexA = 0 #Used to help keep store of the total number of epoches with the models
def generate_batch(batch_size, inputCount):
global data_index, epoch_index
batch = np.ndarray(shape=(batch_size, inputCount), dtype=np.int32)
labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
n=0
while n < batch_size:
if len( set(my_data[data_index, 1]) ) >= inputCount:
labels[n,0] = my_data[data_index, 0]
batch[n] = random.sample( set(my_data[data_index, 1]), inputCount)
n = n+1
data_index = (data_index + 1) % len(my_data) #may have to do something like len my_data[:]
if data_index == 0:
epoch_index = epoch_index + 1
print('Completed %d Epochs' % epoch_index)
else:
data_index = (data_index + 1) % len(my_data)
if data_index == 0:
epoch_index = epoch_index + 1
print('Completed %d Epochs' % epoch_index)
return batch, labels
This is where I define my Estimator and attempt to do training
#Define the estimator
word2vecEstimator = tf.estimator.Estimator(
model_fn=my_model,
params={
'batch_size': 1024,
'embedding_size': 50,
'num_inputs': 5,
'num_sampled':128
})
batch_size = 16
num_inputs = 3
#Train with Estimator
word2vecEstimator.train(
input_fn=generate_batch(batch_size, num_inputs),
steps=10)
This is the error message that I get
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
/usr/lib/python3.6/inspect.py in getfullargspec(func)
1118 skip_bound_arg=False,
-> 1119 sigcls=Signature)
1120 except Exception as ex:
/usr/lib/python3.6/inspect.py in _signature_from_callable(obj, follow_wrapper_chains, skip_bound_arg, sigcls)
2185 if not callable(obj):
-> 2186 raise TypeError('{!r} is not a callable object'.format(obj))
2187
TypeError: (array([[1851833, 670357, 343012],
[ 993526, 431296, 935528],
[ 938067, 1155719, 2277388],
[ 534965, 1125669, 1665716],
[1412657, 2152211, 1176177],
[ 268114, 2097642, 2707258],
[1280762, 1516464, 453615],
[2545980, 2302607, 2421182],
[1706260, 2735027, 292652],
[1802025, 2949676, 653015],
[ 854228, 2626773, 225486],
[1747135, 1608478, 2503487],
[1326661, 272883, 2089444],
[3082922, 1359481, 621031],
[2636832, 1842777, 1979638],
[2512269, 1617986, 389356]], dtype=int32), array([[1175598],
[2528125],
[1870906],
[ 643521],
[2349752],
[ 754986],
[2277570],
[2121120],
[2384306],
[1881398],
[3046987],
[2505729],
[2908573],
[2438025],
[ 441422],
[2355625]], dtype=int32)) is not a callable object
The above exception was the direct cause of the following exception:
TypeError Traceback (most recent call last)
<ipython-input-15-7acc939af001> in <module>()
5 word2vecEstimator.train(
6 input_fn=generate_batch(batch_size, num_inputs),
----> 7 steps=10)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/estimator/estimator.py in train(self, input_fn, hooks, steps, max_steps, saving_listeners)
352
353 saving_listeners = _check_listeners_type(saving_listeners)
--> 354 loss = self._train_model(input_fn, hooks, saving_listeners)
355 logging.info('Loss for final step: %s.', loss)
356 return self
/usr/local/lib/python3.6/dist-packages/tensorflow/python/estimator/estimator.py in _train_model(self, input_fn, hooks, saving_listeners)
1205 return self._train_model_distributed(input_fn, hooks, saving_listeners)
1206 else:
-> 1207 return self._train_model_default(input_fn, hooks, saving_listeners)
1208
1209 def _train_model_default(self, input_fn, hooks, saving_listeners):
/usr/local/lib/python3.6/dist-packages/tensorflow/python/estimator/estimator.py in _train_model_default(self, input_fn, hooks, saving_listeners)
1232 features, labels, input_hooks = (
1233 self._get_features_and_labels_from_input_fn(
-> 1234 input_fn, model_fn_lib.ModeKeys.TRAIN))
1235 worker_hooks.extend(input_hooks)
1236 estimator_spec = self._call_model_fn(
/usr/local/lib/python3.6/dist-packages/tensorflow/python/estimator/estimator.py in _get_features_and_labels_from_input_fn(self, input_fn, mode)
1073 """Extracts the `features` and labels from return values of `input_fn`."""
1074 return estimator_util.parse_input_fn_result(
-> 1075 self._call_input_fn(input_fn, mode))
1076
1077 def _extract_batch_length(self, preds_evaluated):
/usr/local/lib/python3.6/dist-packages/tensorflow/python/estimator/estimator.py in _call_input_fn(self, input_fn, mode)
1151 ValueError: if `input_fn` takes invalid arguments.
1152 """
-> 1153 input_fn_args = function_utils.fn_args(input_fn)
1154 kwargs = {}
1155 if 'mode' in input_fn_args:
/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/function_utils.py in fn_args(fn)
54 if _is_callable_object(fn):
55 fn = fn.__call__
---> 56 args = tf_inspect.getfullargspec(fn).args
57 if _is_bounded_method(fn):
58 args.remove('self')
/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/tf_inspect.py in getfullargspec(obj)
214 return next((d.decorator_argspec
215 for d in decorators
--> 216 if d.decorator_argspec is not None), _getfullargspec(target))
217
218
/usr/lib/python3.6/inspect.py in getfullargspec(func)
1123 # else. So to be fully backwards compatible, we catch all
1124 # possible exceptions here, and reraise a TypeError.
-> 1125 raise TypeError('unsupported callable') from ex
1126
1127 args = []
TypeError: unsupported callable
Here is a link to the Google Colab notebook for people to run on their own. For anyone looking to execute this, this will download a data file that is ~500 mbs.
https://colab.research.google.com/drive/1LjIz04xhRi5Fsw_Q3IzoG_5KkkXI3WFE
And here is the full code, from the notebook.
import math
import numpy as np
import random
import zipfile
import shutil
from collections import namedtuple
import os
import pprint
import tensorflow as tf
import pandas as pd
import pickle
from numpy import genfromtxt
!pip install -U -q PyDrive
from google.colab import files
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
from google.colab import auth
from oauth2client.client import GoogleCredentials
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)
vocabulary_size = 3096637 #updated 10-25-18 3096636
import gc
dl_id = '19yha9Scxq4zOdfPcw5s6L2lkYQWenApC' #updated 10-22-18
myDownload = drive.CreateFile({'id': dl_id})
myDownload.GetContentFile('Data.npy')
my_data = np.load('Data.npy')
#os.remove('Data.npy')
np.random.shuffle(my_data)
print(my_data[0:15])
data_index = 0
epoch_index = 0
recEpoch_indexA = 0 #Used to help keep store of the total number of epoches with the models
def generate_batch(batch_size, inputCount):
global data_index, epoch_index
batch = np.ndarray(shape=(batch_size, inputCount), dtype=np.int32)
labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
n=0
while n < batch_size:
if len( set(my_data[data_index, 1]) ) >= inputCount:
labels[n,0] = my_data[data_index, 0]
batch[n] = random.sample( set(my_data[data_index, 1]), inputCount)
n = n+1
data_index = (data_index + 1) % len(my_data) #may have to do something like len my_data[:]
if data_index == 0:
epoch_index = epoch_index + 1
print('Completed %d Epochs' % epoch_index)
else:
data_index = (data_index + 1) % len(my_data)
if data_index == 0:
epoch_index = epoch_index + 1
print('Completed %d Epochs' % epoch_index)
return batch, labels
def my_model( features, labels, mode, params):
# train_dataset = tf.placeholder(tf.int32, shape=[batch_size, num_inputs ])
# train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
train_dataset = features
train_labels = labels
batch_sizeE=params["batch_size"]
embedding_sizeE=params["embedding_size"]
num_inputsE=params["num_inputs"]
num_sampledE=params["num_sampled"]
epochCount = tf.get_variable( 'epochCount', initializer= 0) #to store epoch count to total # of epochs are known
update_epoch = tf.assign(epochCount, epochCount + 1)
embeddings = tf.get_variable( 'embeddings', dtype=tf.float32,
initializer= tf.random_uniform([vocabulary_size, embedding_sizeE], -1.0, 1.0, dtype=tf.float32) )
softmax_weights = tf.get_variable( 'softmax_weights', dtype=tf.float32,
initializer= tf.truncated_normal([vocabulary_size, embedding_sizeE],
stddev=1.0 / math.sqrt(embedding_sizeE), dtype=tf.float32 ) )
softmax_biases = tf.get_variable('softmax_biases', dtype=tf.float32,
initializer= tf.zeros([vocabulary_size], dtype=tf.float32), trainable=False )
embed = tf.nn.embedding_lookup(embeddings, train_dataset) #train data set is
embed_reshaped = tf.reshape( embed, [batch_sizeE*num_inputs, embedding_sizeE] )
segments= np.arange(batch_size).repeat(num_inputs)
averaged_embeds = tf.segment_mean(embed_reshaped, segments, name=None)
loss = tf.reduce_mean(
tf.nn.sampled_softmax_loss(weights=softmax_weights, biases=softmax_biases, inputs=averaged_embeds,
sampled_values=tf.nn.uniform_candidate_sampler(true_classes=tf.cast(train_labels, tf.int64), num_sampled=num_sampled, num_true=1, unique=True, range_max=vocabulary_size, seed=None),
labels=train_labels, num_sampled=num_sampled, num_classes=vocabulary_size))
optimizer = tf.train.AdagradOptimizer(1.0).minimize(loss)
saver = tf.train.Saver()
#Define the estimator
word2vecEstimator = tf.estimator.Estimator(
model_fn=my_model,
params={
'batch_size': 1024,
'embedding_size': 50,
'num_inputs': 5,
'num_sampled':128
})
batch_size = 16
num_inputs = 3
#Train with Estimator
word2vecEstimator.train(
input_fn=generate_batch(batch_size, num_inputs),
steps=10)

There's no way of correcting the function, cuz it can never be implemented using Tensorflow. The input_fn() function must return Tensors, not numpy arrays, cuz the input_fn() is a function constructing the graph, and it maybe is just called once when building the graph. In this context, the numpy array is just a constant value. It may seem to be weird, But it's the truth. You need to understand the mechanism of Tensorflow: the STATIC COMPUTE GRAPH!

Answer here
Tensorflow error : unsupported callable
train method accepts the input function, so it should be input_fn, not input_fn().