I am experimenting with a binary classifier on images whether it is a bee or not. I have gathered a dataset of 12,000 images of 6 categories, one of which is bees. So I have a column is_bee with values of "0" and "1" matching to "Not a bee" and "It is a bee". I am training the classifier and when I try to apply the trained model on any image, even one of the trained ones, it gives me (almost) exclusively value "0" with 73.11% confidence. My code is as follows:
import pandas as pd
import numpy as np
import sys
import os
import random
from pathlib import Path
import imageio
import skimage
import skimage.io
import skimage.transform
import matplotlib.pyplot as plt
import seaborn as sns
import scipy
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras import optimizers, losses
from keras.models import Sequential
from keras.layers import Dense, Conv2D, Flatten, MaxPool2D, Dropout, BatchNormalization,LeakyReLU
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import ModelCheckpoint, Callback, EarlyStopping, ReduceLROnPlateau
from tensorflow.keras.utils import to_categorical
import tensorflow
np.random.seed(42)
tensorflow.random.set_seed(42)
# Global variables
imgs_folder='images/'
img_width=100
img_height=100
img_channels=3
def read_img(file):
img = skimage.io.imread(file)
img = skimage.transform.resize(img, (img_width, img_height), mode='reflect')
return img[:,:,:img_channels]
images=pd.read_csv('/content/gdrive/MyDrive/Colab Notebooks/beeID/images.csv',
index_col=False, sep=";",
dtype={'type':'category', 'is_bee':'category'})
images['file'] = imgs_folder+images['file']
# Cannot impute nans, drop them
images.dropna(inplace=True)
# Some image files don't exist. Leave only bees with available images.
img_exists = images['file'].apply(lambda f: os.path.exists(f))
images = images[img_exists]
images['type'] = images['type'].astype('category')
images['is_bee'] = images['is_bee'].astype('category')
def split_balance(df, field_name):
train_df, test_df = train_test_split(df, random_state=23)
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=23)
ncat_bal = int(len(train_df)/train_df[field_name].cat.categories.size)
train_df_bal = train_df.groupby(field_name, as_index=False).apply(lambda g: g.sample(ncat_bal, replace=True)).reset_index(drop=True)
return(train_df_bal, val_df, test_df)
def prepare2train(train_p2t_df, val_p2t_df, test_p2t_df, field_name):
# Train data
train_X = np.stack(train_p2t_df['file'].apply(read_img))
#train_y = to_categorical(train_bees[field_name].values)
train_y = pd.get_dummies(train_p2t_df[field_name], drop_first=False)
# Validation during training data to calc val_loss metric
val_X = np.stack(val_p2t_df['file'].apply(read_img))
#val_y = to_categorical(val_bees[field_name].values)
val_y = pd.get_dummies(val_p2t_df[field_name], drop_first=False)
# Test data
test_X = np.stack(test_p2t_df['file'].apply(read_img))
#test_y = to_categorical(test_bees[field_name].values)
test_y = pd.get_dummies(test_p2t_df[field_name], drop_first=False)
# Data augmentation
generator = ImageDataGenerator(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range=180, # randomly rotate images in the range (degrees, 0 to 180)
zoom_range=0.1, # Randomly zoom image
width_shift_range=0.2, # randomly shift images horizontally (fraction of total width)
height_shift_range=0.2, # randomly shift images vertically (fraction of total height)
horizontal_flip=True, # randomly flip images
vertical_flip=True)
generator.fit(train_X)
return (generator, train_X, val_X, test_X, train_y, val_y, test_y)
def eval_model(training, model, test_X, test_y, field_name):
## Trained model analysis and evaluation
f, ax = plt.subplots(2,1, figsize=(5,5))
ax[0].plot(training.history['loss'], label="Loss")
ax[0].plot(training.history['val_loss'], label="Validation loss")
ax[0].set_title('%s: loss' % field_name)
ax[0].set_xlabel('Epoch')
ax[0].set_ylabel('Loss')
ax[0].legend()
# Accuracy
ax[1].plot(training.history['accuracy'], label="Accuracy")
ax[1].plot(training.history['val_accuracy'], label="Validation accuracy")
ax[1].set_title('%s: accuracy' % field_name)
ax[1].set_xlabel('Epoch')
ax[1].set_ylabel('Accuracy')
ax[1].legend()
plt.tight_layout()
plt.show()
# Accuracy by subspecies
test_pred = model.predict(test_X)
acc_by_subspecies = np.logical_and((test_pred > 0.5), test_y).sum()/test_y.sum()
acc_by_subspecies.plot(kind='bar', title='Accuracy by %s' % field_name)
plt.ylabel('Accuracy')
plt.show()
# Print metrics
print("Classification report")
test_pred = np.argmax(test_pred, axis=1)
test_truth = np.argmax(test_y.values, axis=1)
# Loss function and accuracy
test_res = model.evaluate(test_X, test_y.values, verbose=1)
# Split/balance and plot the result
train_bees_bal, val_bees, test_bees = split_balance(bees, 'health')
# Split/balance and plot the result
train_images_bal, val_images, test_images = split_balance(images, 'type')
# Will use balanced dataset as main
train_images = train_images_bal
generator_images, train_images_X, val_images_X, test_images_X, train_images_y, val_images_y, test_images_y = prepare2train(train_images, val_images, test_images, 'is_bee')
# We'll stop training if no improvement after some epochs
earlystopper_images = EarlyStopping(monitor='val_accuracy', patience=20, verbose=1)
# Save the best model during the training
checkpointer_images = ModelCheckpoint('model_images.h5'
,monitor='val_accuracy'
,verbose=1
,save_best_only=True
,save_weights_only=True)
# Build CNN model
model_images=Sequential()
model_images.add(Conv2D(5, kernel_size=3, input_shape=(img_width, img_height,3), activation='relu', padding='same'))
model_images.add(MaxPool2D(2))
model_images.add(Conv2D(10, kernel_size=3, activation='relu', padding='same'))
model_images.add(Dropout(0.25))
model_images.add(Flatten())
model_images.add(Dropout(0.5))
model_images.add(Dense(train_images_y.columns.size, activation='sigmoid', name='preds'))
# show model summary
model_images.summary()
model_images.compile(loss=losses.binary_crossentropy,optimizer='adam',metrics=['accuracy'])
# Train
training_images = model_images.fit_generator(generator_images.flow(train_images_X,train_images_y, batch_size=60)
,epochs=100
,validation_data=(val_images_X, val_images_y)
,steps_per_epoch=10
,callbacks=[earlystopper_images, checkpointer_images])
# Get the best saved weights
model_images.load_weights('model_images.h5')
eval_model(training_images, model_images, test_images_X, test_images_y, 'is_bee')
scores = model_images.evaluate(test_images_X, test_images_y, verbose=0)
print("Accuracy: %.2f%%" % (scores[1]*100))
import numpy as np
from google.colab import files
from keras.preprocessing import image
for fn in os.listdir('/content/bee_imgs'):
# predicting images
path = '/content/bee_imgs/' + fn
img = image.load_img(path, target_size=(100, 100))
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
checkImages = np.vstack([x])
classes_images = model_images.predict(checkImages)
score = tensorflow.nn.softmax(classes_images[0])
print(
"This image {} most likely belongs to {} with a {:.2f} percent confidence."
.format(fn, test_images_y.columns[np.argmax(score)], 100 * np.max(score))
)
The result of the above testing on already trained images (but also occurs when I try it with non trained images) is this:
This image 016_252.png most likely belongs to 0 with a 73.11 percent confidence.
This image 031_117.png most likely belongs to 0 with a 73.11 percent confidence.
This image 019_1026.png most likely belongs to 0 with a 73.11 percent confidence.
This image 008_243.png most likely belongs to 1 with a 73.11 percent confidence.
...
This image 039_016.png most likely belongs to 0 with a 73.11 percent confidence.
This image 022_364.png most likely belongs to 0 with a 73.11 percent confidence.
This image 022_274.png most likely belongs to 0 with a 73.11 percent confidence.
So, despite the fact that the model's accuracy has reached 96% in predicting whether an image is a bee or not, when I apply it on any image of a bee it seldom recognizes it as a bee. Is there something wrong with my model building or when I am trying to apply it?
After removing the softmax function and np.argmax, you should just use the same read_img function that was used during training for predictions and it should be fine.
I want to train my model on 1 MNIST class at a time.
I can load the data with a general loader:
import torch
import torch.nn as nn
import torch.optim as optim
import torchvision.datasets as datasets
import torchvision.transforms as transforms
from torch.autograd import Variable
trans = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
# if not exist, download mnist dataset
root = './data'
train_set = datasets.MNIST(root=root, train=True, transform=trans, download=True)
batch_size = 100
train_loader = torch.utils.data.DataLoader(
dataset=train_set,
batch_size=batch_size,
shuffle=True)
But I'm not sure how to create 10 loaders (1 for each of the classes/digits) from this general loader (or just 10 loaders initially)
A rather simple solution would involve grouping the dataset by truth value, and creating a unique dataloader per group:
...
from torch.utils.data import Subset, DataLoader
subsets = {target: Subset(train_set, [i for i, (x, y) in enumerate(train_set) if y == target]) for _, target in train_set.class_to_idx.items()}
loaders = {target: DataLoader(subset) for target, subset in subsets.items()}
you can then pick out a specific loader based on class index:
class_3_loader = loaders[3]
I am making a neural network model using pytorch.
I built a simple and shallow 3 layer model by referring to the tutorial.
However, training is random despite using the same model and script.
In other words, it can be seen that the loss does not drop about once out of 4, so it is not trained. I don't know why the model is shallow and unstable. I would be grateful if someone with the same experience as me or who has solved the problem can advise.
enter image description here
It's same script running result.
1 out of 4 times don't trained.
but I used same script and model.
The value of the input tensor is the same in both the case of learning and the case of not learning.
my script is under here. and x input shape is [10000, 1]
import os
import pandas as pd
from sklearn.preprocessing import StandardScaler
import numpy as np
import torch.nn as nn
import torch.optim as optim
from sklearn.preprocessing import StandardScaler
import sys
import torch
from sklearn.preprocessing import StandardScaler
import re
os.chdir("...")
F1 = os.listdir(os.getcwd())
print(F1)
df = pd.read_excel('10000.xlsx', sheet_name=1)
Ang_tilt = torch.from_numpy(df['Ang_tilt'].values).unsqueeze(dim=1).float()
x_list = [Ang_tilt]
nb_epochs = 3000
import sys
#from aug_data_processing import *
import torch.nn as nn
import torch.optim as optim
from sklearn.preprocessing import StandardScaler
from matplotlib import pyplot as plt
########################################
####################model
#print(x_list)
net = Net(x_dim=Ang_tilt.size()[1])
criterion = nn.MSELoss()
# optimizer = torch.optim.Adam(model.parameters(), lr=1e-9)
optimizer = torch.optim.SGD(net.parameters(), lr=1e-6, momentum=0.7)
losses = []
################forward
for step in range(nb_epochs + 1):
scaler = StandardScaler()
Ang_tilt = scaler.fit_transform(Ang_tilt)
Ang_tilt = torch.from_numpy(Ang_tilt).float()
#print(x_list[i])
prediction = net(Ang_tilt)
#print(prediction)
loss = criterion(input=prediction, target=y_label)
optimizer.zero_grad()
losses.append(loss.item())
loss.backward()
optimizer.step()
#print(Ang_tilt)
plt.title('3_layer_NN_loss_pre+post')
plt.xlabel('epoch')
plt.ylabel('losses')
plt.plot(range(nb_epochs+1), losses)
plt.show()
torch.save(obj=net, f='aug.pt')
And this is Network
from torch import nn
from torch.nn import functional as F
import torch
import torch
from torch.autograd import Variable
'''
x_dim = dimension을 바로
'''
class Net(nn.Module):
def __init__(self, x_dim):
super(Net, self).__init__()
self.fc1 = nn.Linear(x_dim, 150)
self.fc2 = nn.Linear(150, 100)
self.fc3 = nn.Linear(100, 40)
self.fc4 = nn.Linear(40,1)
self.dropout = nn.Dropout(p=0.5)
torch.nn.init.xavier_uniform_(self.fc1.weight)
torch.nn.init.xavier_uniform_(self.fc2.weight)
torch.nn.init.xavier_uniform_(self.fc3.weight)
torch.nn.init.xavier_uniform_(self.fc4.weight)
def forward(self, x):
x = F.relu(self.fc1(x))
x = F.relu(self.fc2(x))
x = F.relu(self.fc3(x))
x = F.relu(self.fc4(x))
return x
I found it by printing my parameters.
When not training, weight is so low.
so I changed model structrue than solved.
I'm working with some code that classifies the infamous dog vs cat image classification using a ResNet-18 model and I'd like to extend it to be able to classify for greater than two image categories (like dog vs cat vs hamster vs ....). In particular I've got 5 categories. I'm new at transfer learning and I'm not sure what I have to change in my code to make this work.
import torch
import numpy as np
import torch.nn.functional as F
from torch.nn import Linear
from torch.utils.data import DataLoader, random_split
from torch.optim import Adam
from torchvision.transforms import Compose, Resize, ToTensor
from torchvision.datasets import ImageFolder
from torchvision.models import resnet18
from matplotlib import pyplot as plt
import random
transform = Compose([Resize((128,128)), ToTensor()])
ds = ImageFolder("*Image_Folder*", transform=transform)
ds_train, ds_val = random_split(ds, [3250, 1073])
dl_train = DataLoader(ds_train, batch_size= 32, shuffle=True)
dl_val = DataLoader(ds_val, batch_size= len(ds_val), shuffle= True)
model = resnet18(pretrained=True)
model.requires_grad_(False)
model.fc = Linear(model.fc.in_features, 5)
X_val, y_val = next(dl_val.__iter__())
opt = torch.optim.Adam(model.parameters(), lr=0.001)
def accuracy(yy, y):
return torch.mean(1.0*(yy == y))
X_val.shape, y_val.shape
y_val = y_val.reshape(-1, 1).float()
for epoch in range(10):
losses = []
accs = []
losses_val = []
accs_val = []
model.train()
for X, y in dl_train:
y = y.reshape(-1, 1).float()
yy = torch.sigmoid(model(X))
loss = F.binary_cross_entropy(yy, y)
losses.append(loss.item())
loss.backward()
opt.step()
opt.zero_grad()
acc = accuracy(torch.round(yy), y)
accs.append(acc.item())
model.eval()
with torch.no_grad():
yy_val = torch.sigmoid(model(X_val))
loss_val = F.binary_cross_entropy(yy_val, y_val)
losses_val.append(loss_val.item())
acc_val = accuracy(torch.round(yy_val), y_val)
accs_val.append(acc_val.item())
print(f"Epoch {epoch}: t-loss = {np.mean(losses):.4f}, t-acc = {np.mean(accs):.4f}, v-loss = {loss_val:.4f}, v-acc = {acc_val:.4f}")
I believe the code is fine up to the for loop, however it could be something I need to add or alter. Currently the line loss = F.binary_cross_entropy(yy, y) is what's giving me an error ValueError: Using a target size (torch.Size([32, 1])) that is different to the input size (torch.Size([32, 5])) is deprecated. Please ensure they have the same size.
This is the data I'm working from: https://www.kaggle.com/alxmamaev/flowers-recognition
Binary Cross Entropy is a loss function designed for binary classification tasks.
In order to convert this model into one capable of 5-class classification, in addition to changing the final layer's width to 5, you need to change the loss function to a multinomial scorer e.g. CrossEntropyLoss().
I used SubsetRandomSampler to split the training data to train (80%) and validation data (20%). But it is showing the same number of images for both after the split (4996):
>>> print('len(train_data): ', len(train_loader.dataset))
>>> print('len(valid_data): ', len(validation_loader.dataset))
len(train_data): 4996
len(valid_data): 4996
Full code:
import numpy as np
import torch
from torchvision import transforms
from torch.utils.data.sampler import SubsetRandomSampler
train_transforms = transforms.Compose([transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])])
dataset = datasets.ImageFolder( '/data/images/train', transform=train_transforms )
validation_split = .2
shuffle_dataset = True
random_seed= 42
batch_size = 20
dataset_size = len(dataset) #4996
indices = list(range(dataset_size))
split = int(np.floor(validation_split * dataset_size))
if shuffle_dataset :
np.random.seed(random_seed)
np.random.shuffle(indices)
train_indices, val_indices = indices[split:], indices[:split]
train_sampler = SubsetRandomSampler(train_indices)
valid_sampler = SubsetRandomSampler(val_indices)
train_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=train_sampler)
validation_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=valid_sampler)
train_loader.dataset and validation_loader.dataset are methods which return the underlying original dataset that the loaders sample from (i.e. the original dataset of size 4996).
If you iterate through the loaders themselves you will see they only return as many samples (acounting for batching) as you have included in the index for each sampler, however.