I am trying to extract key value pair from scanned invoices document using LayoutLMV2 model but I am getting error. Installation guide. I am just trying to check how the model is predicting the key value pair from the document or do I need to fine tune the model in own data set. Need help in this.
from transformers import PreTrainedTokenizerBase, LayoutLMv2FeatureExtractor, LayoutLMv2Processor, AutoTokenizer, LayoutLMv2ForRelationExtraction, AdamW
from transformers.file_utils import PaddingStrategy
import torch
from torch import nn
from dataclasses import dataclass
from typing import Dict, Tuple, Optional, Union
from datasets import load_dataset
from torch.utils.data import DataLoader
from torchvision.transforms import ToPILImage
from torchvision import transforms
from PIL import Image, ImageFont, ImageDraw, ImageEnhance, ImageFilter
import numpy as np
import pytesseract
pytesseract.pytesseract.tesseract_cmd = r'C:\Users\name\AppData\Local\Programs\Tesseract-OCR\tesseract.exe'
feature_extractor = LayoutLMv2FeatureExtractor(apply_ocr=True)
tokenizer = AutoTokenizer.from_pretrained(path_1, pad_token='<pad>')
processor = LayoutLMv2Processor(feature_extractor, tokenizer)
model = LayoutLMv2ForRelationExtraction.from_pretrained(path_1)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(device)
image_file = 'image2.png'
image = Image.open(image_file).convert('RGB')
image.size
encoded_inputs = processor(image, return_tensors="pt")
encoded_inputs.keys()
for k,v in encoded_inputs.items():
print(k, v.shape)
for k,v in encoded_inputs.items():
encoded_inputs[k] = v.to(model.device)
# forward pass
outputs = model(**encoded_inputs)
This is the error I am getting
TypeError Traceback (most recent call last)
c:\Users\name\Parallel Project\Trans_LayoutXLM.ipynb Cell 7 in <cell line: 5>()
2 encoded_inputs[k] = v.to(model.device)
4 # forward pass
----> 5 outputs = model(**encoded_inputs)
File c:\Users\name\.conda\envs\layoutlmft\lib\site-packages\torch\nn\modules\module.py:1130, in Module._call_impl(self, *input, **kwargs)
1126 # If we don't have any hooks, we want to skip the rest of the logic in
1127 # this function, and just call forward.
1128 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1129 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1130 return forward_call(*input, **kwargs)
1131 # Do not call functions when jit is used
1132 full_backward_hooks, non_full_backward_hooks = [], []
File c:\Users\name\.conda\envs\layoutlmft\lib\site-packages\transformers\models\layoutlmv2\modeling_layoutlmv2.py:1598, in LayoutLMv2ForRelationExtraction.forward(self, input_ids, bbox, labels, image, attention_mask, token_type_ids, position_ids, head_mask, entities, relations)
1596 sequence_output, image_output = outputs[0][:, :seq_length], outputs[0][:, seq_length:]
1597 sequence_output = self.dropout(sequence_output)
-> 1598 loss, pred_relations = self.extractor(sequence_output, entities, relations)
1600 return RegionExtractionOutput(
1601 loss=loss,
1602 entities=entities,
(...)
1605 hidden_states=outputs[0],
1606 )
File c:\Users\name\.conda\envs\layoutlmft\lib\site-packages\torch\nn\modules\module.py:1130, in Module._call_impl(self, *input, **kwargs)
1126 # If we don't have any hooks, we want to skip the rest of the logic in
1127 # this function, and just call forward.
1128 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1129 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1130 return forward_call(*input, **kwargs)
1131 # Do not call functions when jit is used
1132 full_backward_hooks, non_full_backward_hooks = [], []
...
-> 1421 batch_size = len(relations)
1422 new_relations = []
1423 for b in range(batch_size):
TypeError: object of type 'NoneType' has no len()
import numpy as np
from transformers import LayoutLMv2Processor, LayoutLMv2ForTokenClassification
from datasets import load_dataset
from PIL import Image, ImageDraw, ImageFont
processor = LayoutLMv2Processor.from_pretrained("microsoft/layoutlmv2-base-uncased")
model = LayoutLMv2ForTokenClassification.from_pretrained("nielsr/layoutlmv2-finetuned-funsd")
dataset = load_dataset("nielsr/funsd", split="test")
labels = dataset.features['ner_tags'].feature.names
id2label = {v: k for v, k in enumerate(labels)}
label2color = {'question':'blue', 'answer':'green', 'header':'orange', 'other':'red'}
def unnormalize_box(bbox, width, height):
return [
width * (bbox[0] / 1000),
height * (bbox[1] / 1000),
width * (bbox[2] / 1000),
height * (bbox[3] / 1000),
]
def iob_to_label(label):
label = label[2:]
if not label:
return 'other'
return label
image_path="invoice.jpg"
# load image example
image = Image.open(image_path).convert("RGB")
image
def process_image(image):
width, height = image.size
# encode
encoding = processor(image, truncation=True, return_offsets_mapping=True, return_tensors="pt")
offset_mapping = encoding.pop('offset_mapping')
# forward pass
outputs = model(**encoding)
# get predictions
predictions = outputs.logits.argmax(-1).squeeze().tolist()
token_boxes = encoding.bbox.squeeze().tolist()
# only keep non-subword predictions
is_subword = np.array(offset_mapping.squeeze().tolist())[:,0] != 0
true_predictions = [id2label[pred] for idx, pred in enumerate(predictions) if not is_subword[idx]]
true_boxes = [unnormalize_box(box, width, height) for idx, box in enumerate(token_boxes) if not is_subword[idx]]
# draw predictions over the image
draw = ImageDraw.Draw(image)
font = ImageFont.load_default()
for prediction, box in zip(true_predictions, true_boxes):
predicted_label = iob_to_label(prediction).lower()
draw.rectangle(box, outline=label2color[predicted_label])
draw.text((box[0]+10, box[1]-10), text=predicted_label, fill=label2color[predicted_label], font=font)
return image
process_image(image)
Related
I am working on Unsupervised Domain Adaptation and I'm working on a geometric learning approach to this problem. In this approach the a custom model uses the scores of a CNN model (in my case a pretrained ResNet50) and uses the class scores of the original ImageNet to build a graph and applies a GCN to the resulting graph. Here is the code of my forward pass
def forward(self, x):
features = self.cnn.forward(x)
scores = self.dsa.forward(x)
transposed_scores = torch.transpose(scores, 0, 1)
adjacency_matrix = torch.matmul(scores, transposed_scores)
sparse_adj_matrix = dense_to_sparse(adjacency_matrix)
edge_index, edge_attr = sparse_adj_matrix[0], sparse_adj_matrix[1]
graph = geometric_data(scores, edge_index=edge_index)
graph.to(self.device)
gcn_features = self.gcn(graph.x, graph.edge_indexes)
concat_features = torch.cat((features, gcn_features))
domain_classification = self.domain_alignment(concat_features)
pseudo_label = self.classifier(concat_features)
return domain_classification, pseudo_label
The problem is that when calling the forward method, I get a runtime error saying 'GlobalStorage' object has no attribute 'edge_indexes'. I'm working on Google Colab. What did I get wrong?
EDIT:
The full error traceback is:
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
/usr/local/lib/python3.8/dist-packages/torch_geometric/data/storage.py in __getattr__(self, key)
61 try:
---> 62 return self[key]
63 except KeyError:
/usr/local/lib/python3.8/dist-packages/torch_geometric/data/storage.py in __getitem__(self, key)
84 def __getitem__(self, key: str) -> Any:
---> 85 return self._mapping[key]
86
KeyError: 'edge_indexes'
During handling of the above exception, another exception occurred:
AttributeError Traceback (most recent call last)
<ipython-input-32-7436986037c0> in <module>
7 uda_model.to(device)
8
----> 9 summary(uda_model, input_size=(3, 224, 224))
/usr/local/lib/python3.8/dist-packages/torchsummary/torchsummary.py in summary(model, input_size, batch_size, device)
70 # make a forward pass
71 # print(x.shape)
---> 72 model(*x)
73
74 # remove these hooks
/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1192 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1193 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1194 return forward_call(*input, **kwargs)
1195 # Do not call functions when jit is used
1196 full_backward_hooks, non_full_backward_hooks = [], []
<ipython-input-31-1c0d271d6797> in forward(self, x)
43 graph = geometric_data(scores, edge_index=edge_index)
44
---> 45 gcn_features = self.gcn(graph.x, graph.edge_indexes)
46 concat_features = torch.cat((features, gcn_features))
47 domain_classification = self.domain_alignment(concat_features)
/usr/local/lib/python3.8/dist-packages/torch_geometric/data/data.py in __getattr__(self, key)
426 "dataset, remove the 'processed/' directory in the dataset's "
427 "root folder and try again.")
--> 428 return getattr(self._store, key)
429
430 def __setattr__(self, key: str, value: Any):
/usr/local/lib/python3.8/dist-packages/torch_geometric/data/storage.py in __getattr__(self, key)
62 return self[key]
63 except KeyError:
---> 64 raise AttributeError(
65 f"'{self.__class__.__name__}' object has no attribute '{key}'")
66
AttributeError: 'GlobalStorage' object has no attribute 'edge_indexes'
For a minimum reproducible example:
import os
import random
import numpy as np
from PIL import Image
from skimage.metrics import structural_similarity as ssim
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader, random_split
from torchvision import transforms
from torch_geometric import nn as geometric_nn
from torch_geometric.data import Data as geometric_data
from torch_geometric.utils import dense_to_sparse
from torchsummary import summary
The code here was divided into different cells in Google Colab, here reported in different code sections
def get_transform():
transform = list()
transform.append(transforms.Resize((256, 256)))
transform.append(transforms.RandomCrop((224, 224)))
transform.append(transforms.RandomHorizontalFlip(0.5))
transform.append(transforms.RandomVerticalFlip(0.5))
transform.append(transforms.GaussianBlur((3, 3)))
transform.append(transforms.RandomGrayscale(0.5))
transform.append(transforms.ToTensor())
transform.append(transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]))
return transforms.Compose(transform)
class GCANDataset(Dataset):
def __init__(self, source_path, target_path, transform=None):
self.transform = transform
self.source_path = source_path
self.target_path = target_path
self.file_counter = 0
self.source_length = 0
for folder in os.listdir(source_path):
path = os.path.join(source_path, folder)
self.file_counter += len(os.listdir(path))
self.source_length += len(os.listdir(path))
for folder in os.listdir(target_path):
path = os.path.join(target_path, folder)
self.file_counter += len(os.listdir(path))
def __len__(self):
return self.file_counter
def __getitem__(self, index):
path_to_use = self.source_path
domain = 0
if index > self.source_length:
path_to_use = self.target_path
index -= self.source_length
domain = 1
label = ''
image = 0
for folder in os.listdir(path_to_use):
item_list = os.listdir(os.path.join(path_to_use, folder))
if index - len(item_list) < 0:
file_to_take = len(item_list) - index - 1
file_path = os.path.join(os.path.join(path_to_use, folder), item_list[file_to_take])
image = Image.open(file_path)
label = folder
if self.transform is not None:
image = self.transform(image)
return image, label, domain
class GCANModel(nn.Module):
def __init__(self, num_classes, gcn_in_channels=224, gcn_out_channels=150, device='cuda:0'):
super(GCANModel, self).__init__()
self.device = device
self.cnn = resnet50(weights=ResNet50_Weights.IMAGENET1K_V2)
features = self.cnn.fc.in_features
combined_features = features + gcn_out_channels
self.cnn = nn.Sequential(*list(self.cnn.children())[:-1])
self.dsa = resnet50(weights=ResNet50_Weights.IMAGENET1K_V2)
self.gcn = geometric_nn.GCNConv(in_channels=gcn_in_channels,
out_channels=gcn_out_channels)
self.domain_alignment = nn.Sequential(
nn.Linear(in_features=combined_features, out_features=1024),
nn.ReLU(),
nn.Linear(in_features=1024, out_features=1024),
nn.ReLU(),
nn.Linear(in_features=1024, out_features=1),
nn.Sigmoid()
)
self.classifier = nn.Sequential(
nn.Linear(in_features=combined_features, out_features=1024),
nn.Dropout(p=0.2),
nn.ReLU(),
nn.Linear(in_features=1024, out_features=1024),
nn.Dropout(p=0.2),
nn.ReLU(),
nn.Linear(in_features=1024, out_features=num_classes),
nn.Softmax()
)
def forward(self, x):
features = self.cnn.forward(x)
scores = self.dsa.forward(x)
transposed_scores = torch.transpose(scores, 0, 1)
adjacency_matrix = torch.matmul(scores, transposed_scores)
sparse_adj_matrix = dense_to_sparse(adjacency_matrix)
edge_index, edge_attr = sparse_adj_matrix[0], sparse_adj_matrix[1]
graph = geometric_data(scores, edge_index=edge_index)
gcn_features = self.gcn(graph.x, graph.edge_indexes)
concat_features = torch.cat((features, gcn_features))
domain_classification = self.domain_alignment(concat_features)
pseudo_label = self.classifier(concat_features)
return domain_classification, pseudo_label
real_world_path = '/content/Adaptiope/product_images'
products_path = '/content/Adaptiope/real_life'
device='cuda:0'
transform = get_transform()
uda_dataset = GCANDataset(source_path=real_world_path,
target_path=products_path, transform=transform)
uda_model = GCANModel(20)
uda_model.to(device)
summary(uda_model, input_size=(3, 224, 224))
The dataset is Adaptiope, which can be found here. The dataset is divided into 3 domains. The synthetic one is ignored, while on the others only 20 classes are considered. The classes mantained are:
backpack, bookcase, car jack, comb, crown, file cabinet, flat iron, game controller, glasses, helicopter, ice skates, letter tray, monitor, mug, network switch, over-ear headphones, pen, purse, stand mixer, stroller
I am working with GCNN. My input data are in float64. But whenever I run my code, this error is shown. I tried converting all tensors to double and it didn't work. Primarily my data are in numpy array then I converted those into pytorch tensors.
Here is my data. Here I converted numpy arrays into tensors and convert the tensors into geometric data to run gcnn.
e_index1 = torch.tensor(edge_index)
x1 = torch.tensor(x)
y1 = torch.tensor(y)
print(x.dtype)
print(y.dtype)
print(edge_index.dtype)
from torch_geometric.data import Data
data = Data(x=x1, edge_index=e_index1, y=y1)
Output:
float64
float64
int64
Here is my code of gcnn class and the rest of the code.
import torch
import torch.nn.functional as F
from torch_geometric.nn import GCNConv
class GCN(torch.nn.Module):
def __init__(self):
super().__init__()
self.conv1 = GCNConv(data.num_node_features, 16)
self.conv2 = GCNConv(16, data.num_node_features)
def forward(self, data):
x, edge_index = data.x, data.edge_index
x = self.conv1(x, edge_index)
x = F.relu(x)
x = F.dropout(x, training=self.training)
x = self.conv2(x, edge_index)
return F.log_softmax(x, dim=1)
device = torch.device('cpu')
model = GCN().to(device)
data = data.to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=5e-4)
model.train()
for epoch in range(10):
optimizer.zero_grad()
out = model(data)
loss = F.nll_loss(out[data.train_mask], data.y[data.train_mask])
loss.backward()
optimizer.step()
error log
---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
<ipython-input-148-e816c251670b> in <module>
7 for epoch in range(10):
8 optimizer.zero_grad()
----> 9 out = model(data)
10 loss = F.nll_loss(out[data.train_mask], data.y[data.train_mask])
11 loss.backward()
5 frames
/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1188 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1189 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1190 return forward_call(*input, **kwargs)
1191 # Do not call functions when jit is used
1192 full_backward_hooks, non_full_backward_hooks = [], []
<ipython-input-147-c1bfee724570> in forward(self, data)
13 x, edge_index = data.x.type(torch.DoubleTensor), data.edge_index
14
---> 15 x = self.conv1(x, edge_index)
16 x = F.relu(x)
17 x = F.dropout(x, training=self.training)
/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1188 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1189 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1190 return forward_call(*input, **kwargs)
1191 # Do not call functions when jit is used
1192 full_backward_hooks, non_full_backward_hooks = [], []
/usr/local/lib/python3.8/dist-packages/torch_geometric/nn/conv/gcn_conv.py in forward(self, x, edge_index, edge_weight)
193 edge_index = cache
194
--> 195 x = self.lin(x)
196
197 # propagate_type: (x: Tensor, edge_weight: OptTensor)
/usr/local/lib/python3.8/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1188 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1189 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1190 return forward_call(*input, **kwargs)
1191 # Do not call functions when jit is used
1192 full_backward_hooks, non_full_backward_hooks = [], []
/usr/local/lib/python3.8/dist-packages/torch_geometric/nn/dense/linear.py in forward(self, x)
134 x (Tensor): The features.
135 """
--> 136 return F.linear(x, self.weight, self.bias)
137
138 #torch.no_grad()
RuntimeError: expected scalar type Double but found Float
I also tried the given solution in stackover flow blogs. But didn't work. Same error is shown repeatedly.
You can use model.double() to convert all the model parameters into double type. This should give a compatible model given your input data is double. Keep in mind though that double type is usually slower than single due to its higher precision nature.
I've been working on this Segformer (using transformers for image segmentation) I've been having issues with getting a message stating that the target is out of bounds.
So here is the code that I am using.
!pip install --upgrade transformers
!nvidia-smi
from torch.utils.data import Dataset, DataLoader
from transformers import AdamW
import torch
from torch import nn
from sklearn.metrics import accuracy_score
from tqdm.notebook import tqdm
import os
from PIL import Image
from transformers import SegformerForSemanticSegmentation,
SegformerFeatureExtractor
import pandas as pd
import cv2
import numpy as np
import albumentations as aug
os.getcwd()
ok, from here this is my current working directoy:
'/data/users/me/Rugd_dataset'
My images are stored in two folders one called images, the nest mask. Each have a sub directory of test and train
WIDTH = 512
Height = 512
class ImageSegmentationDataset(Dataset):
"""Image segmentation dataset."""
def __init__(self, root_dir, feature_extractor, transforms=None, train=True):
"""
Args:
root_dir (string): Root directory of the dataset containing the images + annotations.
feature_extractor (SegFormerFeatureExtractor): feature extractor to prepare images + segmentation maps.
train (bool): Whether to load "training" or "validation" images + annotations.
"""
self.root_dir = root_dir
self.feature_extractor = feature_extractor
self.train = train
self.transforms = transforms
sub_path = "train" if self.train else "test"
self.img_dir = os.path.join(self.root_dir, "images", sub_path)
self.ann_dir = os.path.join(self.root_dir, "mask", sub_path)
# read images
image_file_names = []
for root, dirs, files in os.walk(self.img_dir):
image_file_names.extend(files)
self.images = sorted(image_file_names)
# read annotations
annotation_file_names = []
for root, dirs, files in os.walk(self.ann_dir):
annotation_file_names.extend(files)
self.annotations = sorted(annotation_file_names)
assert len(self.images) == len(self.annotations), "There must be as many images as there are segmentation maps"
def __len__(self):
return len(self.images)
def __getitem__(self, idx):
image = cv2.imread(os.path.join(self.img_dir,
self.images[idx]))
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
segmentation_map = cv2.imread(os.path.join(self.ann_dir, self.annotations[idx]))
segmentation_map = cv2.cvtColor(segmentation_map, cv2.COLOR_BGR2GRAY)
# image = Image.open()
# segmentation_map = Image.open()
if self.transforms is not None:
augmented = self.transforms(image=image, mask=segmentation_map)
# randomly crop + pad both image and segmentation map to same size
encoded_inputs = self.feature_extractor(augmented['image'], augmented['mask'], return_tensors="pt")
else:
encoded_inputs = self.feature_extractor(image, segmentation_map, return_tensors="pt")
for k,v in encoded_inputs.items():
encoded_inputs[k].squeeze_() # remove batch dimension
return encoded_inputs
transform = aug.Compose([
aug.Flip(p=0.5)
])
dir_path = r'images/train'
count = 0
# Iterate directory
for path in os.listdir(dir_path):
# check if current path is a file
if os.path.isfile(os.path.join(dir_path, path)):
count += 1
print('File count:', count)
current count = File count: 7386
dir_path = r'mask/train'
count = 0
# Iterate directory
for path in os.listdir(dir_path):
# check if current path is a file
if os.path.isfile(os.path.join(dir_path, path)):
count += 1
print('File count:', count)
this is what I receive as a File count: 7386. So my paths to the images seem to fit.
root_dir = '/data/users/christopher.kehl/Rugd_dataset'
feature_extractor = SegformerFeatureExtractor(align=False, reduce_zero_label=False)
train_dataset = ImageSegmentationDataset(root_dir=root_dir, feature_extractor=feature_extractor, transforms=transform)
valid_dataset = ImageSegmentationDataset(root_dir=root_dir, feature_extractor=feature_extractor, transforms=None, train=False)
print("Number of training examples:", len(train_dataset))
print("Number of validation examples:", len(valid_dataset))
Number of training examples: 7387
Number of validation examples: 100
encoded_inputs = train_dataset[-1]
so the index here is supposed to be zero, but I can only get this to work with a 1 or an -1, anything other than 0.
encoded_inputs["pixel_values"].shape
torch.Size([3, 512, 512])
encoded_inputs["labels"].shape
torch.Size([512, 512])
encoded_inputs["labels"].shape
torch.Size([512, 512])
encoded_inputs["labels"].squeeze().unique()
tensor([ 29, 60, 76, 91, 107])
mask = encoded_inputs["labels"].numpy()
import matplotlib.pyplot as plt
plt.imshow(mask)
shows the images of a mask
from torch.utils.data import DataLoader
train_dataloader = DataLoader(train_dataset, batch_size=4, shuffle=True)
valid_dataloader = DataLoader(valid_dataset, batch_size=4)
batch = next(iter(train_dataloader))
for k,v in batch.items():
print(k, v.shape)
pixel_values torch.Size([4, 3, 512, 512])
labels torch.Size([4, 512, 512])
batch["labels"].shape
torch.Size([4, 512, 512])
classes = pd.read_csv('class_dict_seg.csv')['name']
id2label = classes.to_dict()
label2id = {v: k for k, v in id2label.items()}
model = SegformerForSemanticSegmentation.from_pretrained("nvidia/mit-b5", ignore_mismatched_sizes=True,
num_labels=len(id2label), id2label=id2label, label2id=label2id,
reshape_last_stage=True)
optimizer = AdamW(model.parameters(), lr=0.00006)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(device)
print("Model Initialized!")
for epoch in range(1, 11): # loop over the dataset multiple times
print("Epoch:", epoch)
pbar = tqdm(train_dataloader)
accuracies = []
losses = []
val_accuracies = []
val_losses = []
model.train()
for idx, batch in enumerate(pbar):
# get the inputs;
pixel_values = batch["pixel_values"].to(device)
labels = batch["labels"].to(device)
# zero the parameter gradients
optimizer.zero_grad()
# forward
outputs = model(pixel_values=pixel_values, labels=labels)
# evaluate
upsampled_logits = nn.functional.interpolate(outputs.logits, size=labels.shape[-2:], mode="bilinear", align_corners=False)
predicted = upsampled_logits.argmax(dim=1)
mask = (labels != 255) # we don't include the background class in the accuracy calculation
pred_labels = predicted[mask].detach().cpu().numpy()
true_labels = labels[mask].detach().cpu().numpy()
accuracy = accuracy_score(pred_labels, true_labels)
loss = outputs.loss
accuracies.append(accuracy)
losses.append(loss.item())
pbar.set_postfix({'Batch': idx, 'Pixel-wise accuracy': sum(accuracies)/len(accuracies), 'Loss': sum(losses)/len(losses)})
# backward + optimize
loss.backward()
optimizer.step()
else:
model.eval()
with torch.no_grad():
for idx, batch in enumerate(valid_dataloader):
pixel_values = batch["pixel_values"].to(device)
labels = batch["labels"].to(device)
outputs = model(pixel_values=pixel_values, labels=labels)
upsampled_logits = nn.functional.interpolate(outputs.logits, size=labels.shape[-2:], mode="bilinear", align_corners=False)
predicted = upsampled_logits.argmax(dim=1)
mask = (labels != 255) # we don't include the background class in the accuracy calculation
pred_labels = predicted[mask].detach().cpu().numpy()
true_labels = labels[mask].detach().cpu().numpy()
accuracy = accuracy_score(pred_labels, true_labels)
val_loss = outputs.loss
val_accuracies.append(accuracy)
val_losses.append(val_loss.item())
print(f"Train Pixel-wise accuracy: {sum(accuracies)/len(accuracies)}\
Train Loss: {sum(losses)/len(losses)}\
Val Pixel-wise accuracy: {sum(val_accuracies)/len(val_accuracies)}\
Val Loss: {sum(val_losses)/len(val_losses)}")
Here is the error Block:
IndexError: Target 151 is out of bounds.
Full Trace
---------------------------------------------------------------------------
IndexError Traceback (most recent call last)
<ipython-input-32-50d0deee01ff> in <module>
16
17 # forward
---> 18 outputs = model(pixel_values=pixel_values, labels=labels)
19
20 # evaluate
~/.local/lib/python3.6/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1101 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102 return forward_call(*input, **kwargs)
1103 # Do not call functions when jit is used
1104 full_backward_hooks, non_full_backward_hooks = [], []
~/.local/lib/python3.6/site-packages/transformers/models/segformer/modeling_segformer.py in forward(self, pixel_values, labels, output_attentions, output_hidden_states, return_dict)
807 )
808 loss_fct = CrossEntropyLoss(ignore_index=self.config.semantic_loss_ignore_index)
--> 809 loss = loss_fct(upsampled_logits, labels)
810
811 if not return_dict:
~/.local/lib/python3.6/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1101 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102 return forward_call(*input, **kwargs)
1103 # Do not call functions when jit is used
1104 full_backward_hooks, non_full_backward_hooks = [], []
~/.local/lib/python3.6/site-packages/torch/nn/modules/loss.py in forward(self, input, target)
1150 return F.cross_entropy(input, target, weight=self.weight,
1151 ignore_index=self.ignore_index, reduction=self.reduction,
-> 1152 label_smoothing=self.label_smoothing)
1153
1154
~/.local/lib/python3.6/site-packages/torch/nn/functional.py in cross_entropy(input, target, weight, size_average, ignore_index, reduce, reduction, label_smoothing)
2844 if size_average is not None or reduce is not None:
2845 reduction = _Reduction.legacy_get_string(size_average, reduce)
-> 2846 return torch._C._nn.cross_entropy_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index, label_smoothing)
2847
2848
IndexError: Target 150 is out of bounds.
I am trying to use the Pytorch-geometric hypergraph convolution network's layer for training no hypergraph. I am using the Amazon review dataset provided by https://www.cs.cornell.edu/~arb/data/amazon-reviews/
Since a node doesn't have a feature, I merely put constants in all nodes as features. Further for hyperedges, I construct an incidence matrix.
First, download the dataset using the command:
!wget -P /tmp https://github.com/gravitogen/hosting_datasets/releases/download/amazon_review_1.0/amazon-reviews.zip
!unzip /tmp/amazon-reviews.zip
Code is below:
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch_geometric.nn import HypergraphConv
import numpy as np
import pandas as pd
class HyperGraph1(nn.Module):
def __init__(self, nfeatures, nhiddden, nclass, dropout,nlayer=1):
super(HyperGraph1, self).__init__()
self.conv1 = HypergraphConv(nfeatures, 16)
self.conv2 = HypergraphConv(16,nclass)
self.dropout_p = dropout
def forward(self, data):
x, edge_index = data.x, data.edge_index
x = self.conv1(x, edge_index)
x = F.relu(x)
x = F.dropout(x, training=self.training)
x = self.conv2(x, edge_index)
return F.log_softmax(x, dim=1)
folder = 'amazon-reviews/'
node_indices_files = 'node-labels-amazon-reviews.txt'
node_label_files = 'label-names-amazon-reviews.txt'
hyperedge_files = 'hyperedges-amazon-reviews.txt'
node_indices = pd.read_csv(folder + node_indices_files, header=None)
labels_annot = pd.read_csv(folder + node_label_files, header = None)
hyper_edge = []
with open(folder + hyperedge_files) as f:
for line in f.readlines():
chunks = line.split(',')
chunks = [int(i) for i in chunks]
hyper_edge.append(chunks)
nodes = node_indices.index.to_list()
labels = node_indices[node_indices.columns[0]].to_list()
# generating sparse incidence matrix for storing hyperedges
node_on_edge_list = []
edge_indices = []
counter = 0
for l in hyper_edge:
node_on_edge_list += l
edge_indices += [counter]*len(l)
counter = counter + 1
hyperedge_index = torch.tensor([ node_on_edge_list, edge_indices])
# generate `Data` for pytorch-geometric
x = np.zeros(len(nodes)*2, dtype=int)
x = x.reshape(len(nodes),2).tolist()
X = torch.tensor(x, dtype=torch.float)
Y = torch.tensor(labels, dtype=torch.int)
from torch_geometric.data import Data
data = Data(x=X, edge_index=hyperedge_index, y=Y)
# Now get ready for training
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = HyperGraph1(nfeatures=2, nhiddden=16, nclass=labels_annot.shape[0],dropout=0.1).to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=5e-4)
model.train()
random_nums = np.random.rand(1, len(data.y))
train_np = random_nums > 0.5
train_mask = train_np.tolist()[0]
test_np = random_nums <= 0.5
test_mask = test_np.tolist()[0]
data.train_mask = torch.tensor(train_mask)
data.test_mask = torch.tensor(test_mask)
for epoch in range(2000):
optimizer.zero_grad()
out = model(data)
loss = F.nll_loss(out[data.train_mask], data.y[data.train_mask])
loss.backward()
optimizer.step()
However, upon attempt to train the network, I get the following error:
---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
<ipython-input-98-40e1b6f967df> in <module>
1 for epoch in range(2000):
2 optimizer.zero_grad()
----> 3 out = model(data)
4 loss = F.nll_loss(out[data.train_mask], data.y[data.train_mask])
5 loss.backward()
~/anaconda3/envs/gcn/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1049 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1050 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1051 return forward_call(*input, **kwargs)
1052 # Do not call functions when jit is used
1053 full_backward_hooks, non_full_backward_hooks = [], []
<ipython-input-84-0bf0aa6f166c> in forward(self, data)
8 def forward(self, data):
9 x, edge_index = data.x, data.edge_index
---> 10 x = self.conv1(x, edge_index)
11 x = F.relu(x)
12 x = F.dropout(x, training=self.training)
~/anaconda3/envs/gcn/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1049 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1050 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1051 return forward_call(*input, **kwargs)
1052 # Do not call functions when jit is used
1053 full_backward_hooks, non_full_backward_hooks = [], []
~/anaconda3/envs/gcn/lib/python3.7/site-packages/torch_geometric/nn/conv/hypergraph_conv.py in forward(self, x, hyperedge_index, hyperedge_weight, hyperedge_attr)
139
140 D = scatter_add(hyperedge_weight[hyperedge_index[1]],
--> 141 hyperedge_index[0], dim=0, dim_size=num_nodes)
142 D = 1.0 / D
143 D[D == float("inf")] = 0
~/anaconda3/envs/gcn/lib/python3.7/site-packages/torch_scatter/scatter.py in scatter_add(src, index, dim, out, dim_size)
27 out: Optional[torch.Tensor] = None,
28 dim_size: Optional[int] = None) -> torch.Tensor:
---> 29 return scatter_sum(src, index, dim, out, dim_size)
30
31
~/anaconda3/envs/gcn/lib/python3.7/site-packages/torch_scatter/scatter.py in scatter_sum(src, index, dim, out, dim_size)
19 size[dim] = int(index.max()) + 1
20 out = torch.zeros(size, dtype=src.dtype, device=src.device)
---> 21 return out.scatter_add_(dim, index, src)
22 else:
23 return out.scatter_add_(dim, index, src)
RuntimeError: index 2268264 is out of bounds for dimension 0 with size 2268264
I am clueless about how to fix this.Note that 2268264 is the number of nodes in the graph.
I mimicked the dataset construction from https://pytorch-geometric.readthedocs.io/en/latest/notes/introduction.html so I am not sure what mistake I did here.
I am trying to convert my Tensorflow graph to use a custom tensorflow estimator, but I am getting stuck defining the function for input_fn ; I am currently getting an error.
This is the function I use to generate my input data and labels
data_index = 0
epoch_index = 0
recEpoch_indexA = 0 #Used to help keep store of the total number of epoches with the models
def generate_batch(batch_size, inputCount):
global data_index, epoch_index
batch = np.ndarray(shape=(batch_size, inputCount), dtype=np.int32)
labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
n=0
while n < batch_size:
if len( set(my_data[data_index, 1]) ) >= inputCount:
labels[n,0] = my_data[data_index, 0]
batch[n] = random.sample( set(my_data[data_index, 1]), inputCount)
n = n+1
data_index = (data_index + 1) % len(my_data) #may have to do something like len my_data[:]
if data_index == 0:
epoch_index = epoch_index + 1
print('Completed %d Epochs' % epoch_index)
else:
data_index = (data_index + 1) % len(my_data)
if data_index == 0:
epoch_index = epoch_index + 1
print('Completed %d Epochs' % epoch_index)
return batch, labels
This is where I define my Estimator and attempt to do training
#Define the estimator
word2vecEstimator = tf.estimator.Estimator(
model_fn=my_model,
params={
'batch_size': 1024,
'embedding_size': 50,
'num_inputs': 5,
'num_sampled':128
})
batch_size = 16
num_inputs = 3
#Train with Estimator
word2vecEstimator.train(
input_fn=generate_batch(batch_size, num_inputs),
steps=10)
This is the error message that I get
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
/usr/lib/python3.6/inspect.py in getfullargspec(func)
1118 skip_bound_arg=False,
-> 1119 sigcls=Signature)
1120 except Exception as ex:
/usr/lib/python3.6/inspect.py in _signature_from_callable(obj, follow_wrapper_chains, skip_bound_arg, sigcls)
2185 if not callable(obj):
-> 2186 raise TypeError('{!r} is not a callable object'.format(obj))
2187
TypeError: (array([[1851833, 670357, 343012],
[ 993526, 431296, 935528],
[ 938067, 1155719, 2277388],
[ 534965, 1125669, 1665716],
[1412657, 2152211, 1176177],
[ 268114, 2097642, 2707258],
[1280762, 1516464, 453615],
[2545980, 2302607, 2421182],
[1706260, 2735027, 292652],
[1802025, 2949676, 653015],
[ 854228, 2626773, 225486],
[1747135, 1608478, 2503487],
[1326661, 272883, 2089444],
[3082922, 1359481, 621031],
[2636832, 1842777, 1979638],
[2512269, 1617986, 389356]], dtype=int32), array([[1175598],
[2528125],
[1870906],
[ 643521],
[2349752],
[ 754986],
[2277570],
[2121120],
[2384306],
[1881398],
[3046987],
[2505729],
[2908573],
[2438025],
[ 441422],
[2355625]], dtype=int32)) is not a callable object
The above exception was the direct cause of the following exception:
TypeError Traceback (most recent call last)
<ipython-input-15-7acc939af001> in <module>()
5 word2vecEstimator.train(
6 input_fn=generate_batch(batch_size, num_inputs),
----> 7 steps=10)
/usr/local/lib/python3.6/dist-packages/tensorflow/python/estimator/estimator.py in train(self, input_fn, hooks, steps, max_steps, saving_listeners)
352
353 saving_listeners = _check_listeners_type(saving_listeners)
--> 354 loss = self._train_model(input_fn, hooks, saving_listeners)
355 logging.info('Loss for final step: %s.', loss)
356 return self
/usr/local/lib/python3.6/dist-packages/tensorflow/python/estimator/estimator.py in _train_model(self, input_fn, hooks, saving_listeners)
1205 return self._train_model_distributed(input_fn, hooks, saving_listeners)
1206 else:
-> 1207 return self._train_model_default(input_fn, hooks, saving_listeners)
1208
1209 def _train_model_default(self, input_fn, hooks, saving_listeners):
/usr/local/lib/python3.6/dist-packages/tensorflow/python/estimator/estimator.py in _train_model_default(self, input_fn, hooks, saving_listeners)
1232 features, labels, input_hooks = (
1233 self._get_features_and_labels_from_input_fn(
-> 1234 input_fn, model_fn_lib.ModeKeys.TRAIN))
1235 worker_hooks.extend(input_hooks)
1236 estimator_spec = self._call_model_fn(
/usr/local/lib/python3.6/dist-packages/tensorflow/python/estimator/estimator.py in _get_features_and_labels_from_input_fn(self, input_fn, mode)
1073 """Extracts the `features` and labels from return values of `input_fn`."""
1074 return estimator_util.parse_input_fn_result(
-> 1075 self._call_input_fn(input_fn, mode))
1076
1077 def _extract_batch_length(self, preds_evaluated):
/usr/local/lib/python3.6/dist-packages/tensorflow/python/estimator/estimator.py in _call_input_fn(self, input_fn, mode)
1151 ValueError: if `input_fn` takes invalid arguments.
1152 """
-> 1153 input_fn_args = function_utils.fn_args(input_fn)
1154 kwargs = {}
1155 if 'mode' in input_fn_args:
/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/function_utils.py in fn_args(fn)
54 if _is_callable_object(fn):
55 fn = fn.__call__
---> 56 args = tf_inspect.getfullargspec(fn).args
57 if _is_bounded_method(fn):
58 args.remove('self')
/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/tf_inspect.py in getfullargspec(obj)
214 return next((d.decorator_argspec
215 for d in decorators
--> 216 if d.decorator_argspec is not None), _getfullargspec(target))
217
218
/usr/lib/python3.6/inspect.py in getfullargspec(func)
1123 # else. So to be fully backwards compatible, we catch all
1124 # possible exceptions here, and reraise a TypeError.
-> 1125 raise TypeError('unsupported callable') from ex
1126
1127 args = []
TypeError: unsupported callable
Here is a link to the Google Colab notebook for people to run on their own. For anyone looking to execute this, this will download a data file that is ~500 mbs.
https://colab.research.google.com/drive/1LjIz04xhRi5Fsw_Q3IzoG_5KkkXI3WFE
And here is the full code, from the notebook.
import math
import numpy as np
import random
import zipfile
import shutil
from collections import namedtuple
import os
import pprint
import tensorflow as tf
import pandas as pd
import pickle
from numpy import genfromtxt
!pip install -U -q PyDrive
from google.colab import files
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
from google.colab import auth
from oauth2client.client import GoogleCredentials
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)
vocabulary_size = 3096637 #updated 10-25-18 3096636
import gc
dl_id = '19yha9Scxq4zOdfPcw5s6L2lkYQWenApC' #updated 10-22-18
myDownload = drive.CreateFile({'id': dl_id})
myDownload.GetContentFile('Data.npy')
my_data = np.load('Data.npy')
#os.remove('Data.npy')
np.random.shuffle(my_data)
print(my_data[0:15])
data_index = 0
epoch_index = 0
recEpoch_indexA = 0 #Used to help keep store of the total number of epoches with the models
def generate_batch(batch_size, inputCount):
global data_index, epoch_index
batch = np.ndarray(shape=(batch_size, inputCount), dtype=np.int32)
labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
n=0
while n < batch_size:
if len( set(my_data[data_index, 1]) ) >= inputCount:
labels[n,0] = my_data[data_index, 0]
batch[n] = random.sample( set(my_data[data_index, 1]), inputCount)
n = n+1
data_index = (data_index + 1) % len(my_data) #may have to do something like len my_data[:]
if data_index == 0:
epoch_index = epoch_index + 1
print('Completed %d Epochs' % epoch_index)
else:
data_index = (data_index + 1) % len(my_data)
if data_index == 0:
epoch_index = epoch_index + 1
print('Completed %d Epochs' % epoch_index)
return batch, labels
def my_model( features, labels, mode, params):
# train_dataset = tf.placeholder(tf.int32, shape=[batch_size, num_inputs ])
# train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
train_dataset = features
train_labels = labels
batch_sizeE=params["batch_size"]
embedding_sizeE=params["embedding_size"]
num_inputsE=params["num_inputs"]
num_sampledE=params["num_sampled"]
epochCount = tf.get_variable( 'epochCount', initializer= 0) #to store epoch count to total # of epochs are known
update_epoch = tf.assign(epochCount, epochCount + 1)
embeddings = tf.get_variable( 'embeddings', dtype=tf.float32,
initializer= tf.random_uniform([vocabulary_size, embedding_sizeE], -1.0, 1.0, dtype=tf.float32) )
softmax_weights = tf.get_variable( 'softmax_weights', dtype=tf.float32,
initializer= tf.truncated_normal([vocabulary_size, embedding_sizeE],
stddev=1.0 / math.sqrt(embedding_sizeE), dtype=tf.float32 ) )
softmax_biases = tf.get_variable('softmax_biases', dtype=tf.float32,
initializer= tf.zeros([vocabulary_size], dtype=tf.float32), trainable=False )
embed = tf.nn.embedding_lookup(embeddings, train_dataset) #train data set is
embed_reshaped = tf.reshape( embed, [batch_sizeE*num_inputs, embedding_sizeE] )
segments= np.arange(batch_size).repeat(num_inputs)
averaged_embeds = tf.segment_mean(embed_reshaped, segments, name=None)
loss = tf.reduce_mean(
tf.nn.sampled_softmax_loss(weights=softmax_weights, biases=softmax_biases, inputs=averaged_embeds,
sampled_values=tf.nn.uniform_candidate_sampler(true_classes=tf.cast(train_labels, tf.int64), num_sampled=num_sampled, num_true=1, unique=True, range_max=vocabulary_size, seed=None),
labels=train_labels, num_sampled=num_sampled, num_classes=vocabulary_size))
optimizer = tf.train.AdagradOptimizer(1.0).minimize(loss)
saver = tf.train.Saver()
#Define the estimator
word2vecEstimator = tf.estimator.Estimator(
model_fn=my_model,
params={
'batch_size': 1024,
'embedding_size': 50,
'num_inputs': 5,
'num_sampled':128
})
batch_size = 16
num_inputs = 3
#Train with Estimator
word2vecEstimator.train(
input_fn=generate_batch(batch_size, num_inputs),
steps=10)
There's no way of correcting the function, cuz it can never be implemented using Tensorflow. The input_fn() function must return Tensors, not numpy arrays, cuz the input_fn() is a function constructing the graph, and it maybe is just called once when building the graph. In this context, the numpy array is just a constant value. It may seem to be weird, But it's the truth. You need to understand the mechanism of Tensorflow: the STATIC COMPUTE GRAPH!
Answer here
Tensorflow error : unsupported callable
train method accepts the input function, so it should be input_fn, not input_fn().