Related
I am facing a weird problem. I am training my TF model using custom training loops. If I use only dense layers as my 1st layer, the model does not seem to train (I am using flattened MNIST dataset
to train). If I use use a flattened layer on top of my already flattend dataset, the model training seems to be working fine.
Note - The reason why I am using a flattened layer on an already flattend dataset is to show that only using dense layer as the first layer is not working. If I use a conv2d layer on non flattened data, then also model trains perfectly. The issue seems to lie on the dense layers for some reason.
Can't seem to find the issue.
Tensorflow version - 2.9.1
Python version - 3.8.6
Model that works
class CustomModel(keras.Model):
def __init__(self, num_classes, name = None):
super().__init__(name = name)
self._flatten = tf.keras.layers.Flatten()
self._dense1 = tf.keras.layers.Dense(64)
self._dense2 = tf.keras.layers.Dense(num_classes)
#tf.function
def call(self, X, training=False):
X = self._flatten(X)
X = tf.nn.relu(self._dense1(X))
return self._dense2(X)
Model that does not work
class CustomModel(keras.Model):
def __init__(self, num_classes, name = None):
super().__init__(name = name)
self._dense1 = tf.keras.layers.Dense(64)
self._dense2 = tf.keras.layers.Dense(num_classes)
#tf.function
def call(self, X, training=False):
X = tf.nn.relu(self._dense1(X))
return self._dense2(X)
Dataset Used
import tensorflow_datasets as tfds
(ds_train, ds_test), ds_info = tfds.load(
"mnist",
split = ["train", "test"],
shuffle_files = True,
as_supervised = True,
with_info = True
)
def normalize_img(image, label):
return tf.cast(image, tf.float32) / 255.0, label
def flatten_img(image, label):
return tf.reshape(image, [-1, 28 * 28]), label
AUTOTUNE = tf.data.experimental.AUTOTUNE
BATCH_SIZE = 64
# # Train Dataset
ds_train = ds_train.map(normalize_img, num_parallel_calls = AUTOTUNE)
ds_train = ds_train.map(flatten_img, num_parallel_calls = AUTOTUNE)
ds_train = ds_train.cache()
ds_train = ds_train.shuffle(ds_info.splits["train"].num_examples)
ds_train = ds_train.batch(BATCH_SIZE)
ds_train = ds_train.prefetch(AUTOTUNE)
# # Test Dataset
ds_test = ds_test.map(normalize_img, num_parallel_calls = AUTOTUNE)
ds_test = ds_test.map(flatten_img, num_parallel_calls = AUTOTUNE)
ds_test = ds_test.batch(BATCH_SIZE)
ds_test = ds_test.prefetch(AUTOTUNE)
Custom Training Loop
model = CustomModel(10)
num_epochs = 5
optimizer = keras.optimizers.Adam()
loss_fn = keras.losses.SparseCategoricalCrossentropy(from_logits=True)
acc_metric = keras.metrics.SparseCategoricalAccuracy()
#tf.function
def train_epoch(x, y):
with tf.GradientTape() as tape:
y_pred = model(x, training = True)
loss = loss_fn(y, y_pred)
# Getting Gradients
gradients = tape.gradient(loss, model.trainable_weights)
# Back Prop
optimizer.apply_gradients(zip(gradients, model.trainable_weights))
acc_metric.update_state(y, y_pred)
return loss
# Training Loop
for epoch in range(num_epochs):
print(f"\nStart of Training Epoch {epoch + 1}")
for batch_idx, (x_batch, y_batch) in tqdm(enumerate(ds_train), total=len(ds_train)):
loss = train_epoch(x_batch, y_batch)
print(f"Accuracy :- {acc_metric.result()}, Loss :- {loss}")
acc_metric.reset_states()
Flatten Layer Model Stats
No Flatten Layer Model Stats
There are two errors in your flatten_img function. First of all, as Frightera pointed out, you don't need to divide by 255 again. Second, you have one dimension too many. Your flatten_img function outputs a tensor with a shape of (1, 784) instead of (784). That's the reason why the model needs a Flatten() layer in order to work. Replace your code with this and it will work just fine:
return tf.reshape(image, [28 * 28]), label
My model has two inputs and I want to calculate the loss of the two inputs separately because the loss of input 2 has to be multiplied by a weight. Then add up these two losses as the final loss for the model. The structure is somehow like this:
This is my model:
def final_loss(y_true, y_pred):
loss = x_loss_value.output + y_model.output*weight
return loss
def mymodel(input_shape): #pooling=max or avg
img_input1 = Input(shape=(input_shape[0], input_shape[1], input_shape[2], ))
image_input2 = Input(shape=(input_shape[0], input_shape[1], input_shape[2], ))
#for input1
x = Conv2D(32, (3, 3), strides=(2, 2))(img_input1)
x_dense = Dense(2, activation='softmax', name='predictions')(x)
x_loss_value = my_categorical_crossentropy_layer(x)[input1_y_true, input1_y_pred]
x_model = Model(inputs=img_input1, outputs=x_loss_value)
#for input2
y = Conv2D(32, (3, 3), strides=(2, 2))(image_input2)
y_dense = Dense(2, activation='softmax', name='predictions')(y)
y_loss_value = my_categorical_crossentropy_layer(y)[input2_y_true, input2_y_pred]
y_model = Model(inputs=img_input2, outputs=y_loss_value)
concat = concatenate([x_model.output, y_model.output])
final_dense = Dense(2, activation='softmax')(concat)
# Create model.
model = Model(inputs=[img_input1,image_input2], output = final_dense)
return model
model.compile(optimizer = optimizers.adam(lr=1e-7), loss = final_loss, metrics = ['accuracy'])
Most of the related solutions I found just customize the final loss and change the loss in Model.complie(loss=customize_loss).
However, I need to apply different losses for different inputs. I'm trying to use a customized layer like this, and get my loss value for final the loss calculation:
class my_categorical_crossentropy_layer1(Layer):
def __init__(self, **kwargs):
self.is_placeholder = True
super(my_categorical_crossentropy_layer1, self).__init__(**kwargs)
def my_categorical_crossentropy_loss(self, y_true, y_pred):
y_pred = K.constant(y_pred) if not K.is_tensor(y_pred) else y_pred
y_true = K.cast(y_true, y_pred.dtype)
return K.categorical_crossentropy(y_true, y_pred, from_logits=from_logits)
def call(self, y_true, y_pred):
loss = self.my_categorical_crossentropy_loss(y_true, y_pred)
self.add_loss(loss, inputs=(y_true, y_pred))
return loss
But, inside the keras model, I can't figure out how to get the y_true and y_pred of the current epoch/batch for my loss layer.
So I can't add x = my_categorical_crossentropy_layer()[y_true, y_pred] to my model.
Is there any way to do the variable calculation like this in the keras model?
Further, can Keras get the previous epoch's training loss or val loss during training process?
I want to apply the previous epoch's training loss as my weight in the final loss.
this is my proposal...
your it's a double binary classification problem that you want to carry out using a single fit. the first thing to notice is that you need to take care of dimensionality: your input is 4d while your target is 2d one-hot encoded so your network needs something to reduce dimensionality, for example, flatten or global pooling. after this, you can start fitting creating a single model with two inputs and two outputs and use two losses. in your case, the losses are weighted categorical_crossentropy. keras enable by default to set the loss weights using loss_weights parameters. to reproduce the formula loss1*1+loss2*W set the weights to [1, W]. you can use the loss_weights parameter also specifying different losses for your output in this way losses=[loss1, loss2, ....] which are linearly combined with the weights specified in the loss_weights
below a working example
input_shape = (28,28,3)
n_sample = 10
# create dummy data
X1 = np.random.uniform(0,1, (n_sample,)+input_shape) # 4d
X2 = np.random.uniform(0,1, (n_sample,)+input_shape) # 4d
y1 = tf.keras.utils.to_categorical(np.random.randint(0,2, n_sample)) # 2d
y2 = tf.keras.utils.to_categorical(np.random.randint(0,2, n_sample)) # 2d
def mymodel(input_shape, weight):
img_input1 = Input(shape=(input_shape[0], input_shape[1], input_shape[2], ))
img_input2 = Input(shape=(input_shape[0], input_shape[1], input_shape[2], ))
# for input1
x = Conv2D(32, (3, 3), strides=(2, 2))(img_input1)
x = GlobalMaxPool2D()(x) # pass from 4d to 2d
x = Dense(2, activation='softmax', name='predictions1')(x)
# for input2
y = Conv2D(32, (3, 3), strides=(2, 2))(img_input2)
y = GlobalMaxPool2D()(y) # pass from 4d to 2d
y = Dense(2, activation='softmax', name='predictions2')(y)
# Create model
model = Model([img_input1,img_input2], [x,y])
model.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['accuracy'],
loss_weights=[1,weight])
return model
weight = 0.3
model = mymodel(input_shape, weight)
model.summary()
model.fit([X1,X2], [y1,y2], epochs=2)
I'm training a model using TensorFlow 2.0 using tf.GradientTape(), but I find that the model's accuracy is 95% if I use tf.keras.losses.BinaryCrossentropy, but degrade to 75% if I use tf.keras.losses.binary_crossentropy. So I'm confused about the difference about the same metric here?
import pandas as pd
import numpy as np
import tensorflow as tf
from tensorflow.keras import layers
from sklearn.model_selection import train_test_split
def read_data():
red_wine = pd.read_csv("https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv", sep=";")
white_wine = pd.read_csv("https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-white.csv", sep=";")
red_wine["type"] = 1
white_wine["type"] = 0
wines = red_wine.append(white_wine)
return wines
def get_x_y(df):
x = df.iloc[:, :-1].values.astype(np.float32)
y = df.iloc[:, -1].values.astype(np.int32)
return x, y
def build_model():
inputs = layers.Input(shape=(12,))
dense1 = layers.Dense(12, activation="relu", name="dense1")(inputs)
dense2 = layers.Dense(9, activation="relu", name="dense2")(dense1)
outputs = layers.Dense(1, activation = "sigmoid", name="outputs")(dense2)
model = tf.keras.Model(inputs=inputs, outputs=outputs)
return model
def generate_dataset(df, batch_size=32, shuffle=True, train_or_test = "train"):
x, y = get_x_y(df)
ds = tf.data.Dataset.from_tensor_slices((x, y))
if shuffle:
ds = ds.shuffle(10000)
if train_or_test == "train":
ds = ds.batch(batch_size)
else:
ds = ds.batch(len(df))
return ds
# loss_object = tf.keras.losses.binary_crossentropy
loss_object = tf.keras.losses.BinaryCrossentropy()
optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)
def train_step(model, optimizer, x, y):
with tf.GradientTape() as tape:
pred = model(x, training=True)
loss = loss_object(y, pred)
grads = tape.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(zip(grads, model.trainable_variables))
def train_model(model, train_ds, epochs=10):
for epoch in range(epochs):
print(epoch)
for x, y in train_ds:
train_step(model, optimizer, x, y)
def main():
data = read_data()
train, test = train_test_split(data, test_size=0.2, random_state=23)
train_ds = generate_dataset(train, 32, True, "train")
test_ds = generate_dataset(test, 32, False, "test")
model = build_model()
train_model(model, train_ds, 10)
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy']
)
model.evaluate(test_ds)
main()
They should indeed work the same; BinaryCrossentropy uses binary_crossentropy, with difference apparent in docstring descriptions; former's intended for two class labels, whereas later supports an arbitrary class count. However, if passing in targets in expected format, both apply same preprocessing before calling backend's binary_crossentropy, which does the actual computing.
The difference you observe is likely a reproducibility issue; ensure you set the random seed - see function below. For a more complete answer on reproducibility, see here.
Function
def reset_seeds(reset_graph_with_backend=None):
if reset_graph_with_backend is not None:
K = reset_graph_with_backend
K.clear_session()
tf.compat.v1.reset_default_graph()
print("KERAS AND TENSORFLOW GRAPHS RESET") # optional
np.random.seed(1)
random.seed(2)
tf.compat.v1.set_random_seed(3)
print("RANDOM SEEDS RESET") # optional
Usage:
import tensorflow as tf
import tensorflow.keras.backend as K
reset_seeds(K)
Thanks, I find the reasons of the inconsistent accuracy:
The shape of outputs in the model is (None, 1), but the feeded label is (None, ), which cause a wrong meaning with python's broadcast mechanism.
In the source code of tf.keras.losses.BinaryCrossentropy(), while calculating the loss, both y_pred and y_true are processed through a function called squeeze_or_expand_dimensions, which is lacked in tf.keras.losses.binary_crossentropy.
Note: Take care that whether the shape is consistent between input data and model outputs.
I wish to view the final output of training a tf.keras model. In this case it would be an array of predictions from the softmax function, e.g. [0,0,0,1,0,1].
Other threads on here have suggested using model.predict(training_data), but this won't work for my situation since I am using dropout at training and validation, so neurons are randomly dropped and predicting again with the same data will give a different result.
def get_model():
inputs = tf.keras.layers.Input(shape=(input_dims,))
x = tf.keras.layers.Dropout(rate=dropout_rate)(inputs, training=True)
x = tf.keras.layers.Dense(units=29, activation='relu')(x)
x = tf.keras.layers.Dropout(rate=dropout_rate)(x, training=True)
x = tf.keras.layers.Dense(units=15, activation='relu')(x)
outputs = tf.keras.layers.Dense(2, activation='softmax')(x)
model = tf.keras.Model(inputs=inputs, outputs=outputs)
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'])
return model
myModel = get_model()
myModel.summary()
myModel.fit(X_train, y_train,
batch_size = batch_size,
epochs= epochs,
verbose = 1,
validation_data = (X_val, y_val))
In tensorflow, you can grab the output of a model after training quite easily. Here is an example from a Github repo:
input = tf.placeholder(tf.float32, shape=[None, INPUT_DIMS])
labels = tf.placeholder(tf.float32, shape=[None])
hidden = tf.nn.tanh(make_nn_layer(normalized, NUM_HIDDEN))
logits = make_nn_layer(hidden, NUM_CLASSES)
outputs = tf.argmax(logits, 1)
int_labels = tf.to_int64(labels)
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, int_labels, name='xentropy')
train_step = tf.train.AdamOptimizer().minimize(cross_entropy)
correct_prediction = tf.equal(outputs, int_labels)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
validation_dict = {
input: validation_data[:,0:7],
labels: validation_data[:,7],}
for i in range(NUM_BATCHES):
batch = training_data[numpy.random.choice(training_size, BATCH_SIZE, False),:]
train_step.run({input: batch[:,0:7], labels: batch[:,7]})
if i % 100 == 0 or i == NUM_BATCHES - 1:
print('Accuracy %.2f%% at step %d' % (accuracy.eval(validation_dict) * 100, i))
output_data = outputs.eval({input: data_vector[:,0:7]})
The only output I can get from the trained model appears to be a history object. There is also a myModel.output object, but it is a tensor that I can't evaluate without putting data into it. Any ideas?
As far as I know, you can't turn off the dropout after passing training=True when calling the layers (unless you transfer the weights to a new model with the same architecture). However, instead you can build and train your model in normal case (i.e. without using training argument in the calls) and then selectively turn on and off the dropout layer in test phase by defining a backend function (i.e. keras.backend.function()) and setting the learning phase (i.e. keras.backend.learning_phase()):
# build your model normally (i.e. without using `training=True` argument)
# train your model...
from keras import backend as K
func = K.function(model.inputs + [K.learning_phase()], model.outputs)
# run the model with dropout layers being active, i.e. learning_phase == 1
preds = func(list_of_input_arrays + [1])
# run the model with dropout layers being inactive, i.e. learning_phase == 0
preds = func(list_of_input_arrays + [0])
Update: As I suggested above, another approach is to define a new model with the same architecture but without setting training=True, and then transfer the weights from the trained model to this new model. To achieve this, I just add a training argument to your get_model() function:
def get_model(training=None):
inputs = tf.keras.layers.Input(shape=(input_dims,))
x = tf.keras.layers.Dropout(rate=dropout_rate)(inputs, training=training)
x = tf.keras.layers.Dense(units=29, activation='relu')(x)
x = tf.keras.layers.Dropout(rate=dropout_rate)(x, training=training)
x = tf.keras.layers.Dense(units=15, activation='relu')(x)
outputs = tf.keras.layers.Dense(2, activation='softmax')(x)
model = tf.keras.Model(inputs=inputs, outputs=outputs)
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'])
return model
# build a model with dropout layers active in both training and test phases
myModel = get_model(training=True)
# train the model
myModel.fit(...)
# build a clone of the model with dropouts deactivated in test phase
myTestModel = get_model() # note: the `training` is `None` by default
# transfer the weights from the trained model to this model
myTestModel.set_weights(myModel.get_weights())
# use the new model in test phase; the dropouts would not be active
myTestModel.predict(...)
For several days now, I'm trying to replicate my keras training results with pytorch. Whatever I do, the pytorch model will overfit far earlier and stronger to the validation set then in keras. For pytorch I use the same XCeption Code from https://github.com/Cadene/pretrained-models.pytorch.
The dataloading, the augmentation, the validation, the training schedule etc. are equivalent. Am I missing something obvious? There must be a general problem somewhere. I tried thousands of different module constellations, but nothing seems to come even close to the keras training. Can somebody help?
Keras model: val accuracy > 90%
# base model
base_model = applications.Xception(weights='imagenet', include_top=False, input_shape=(img_width, img_height, 3))
# top model
x = base_model.output
x = GlobalMaxPooling2D()(x)
x = Dense(512, activation='relu')(x)
x = Dropout(0.5)(x)
predictions = Dense(4, activation='softmax')(x)
# this is the model we will train
model = Model(inputs=base_model.input, outputs=predictions)
# Compile model
from keras import optimizers
adam = optimizers.Adam(lr=0.0001)
model.compile(loss='categorical_crossentropy',
optimizer=adam, metrics=['accuracy'])
# LROnPlateau etc. with equivalent settings as pytorch
Pytorch model: val accuracy ~81%
from xception import xception
import torch.nn.functional as F
# modified from https://github.com/Cadene/pretrained-models.pytorch
class XCeption(nn.Module):
def __init__(self, num_classes):
super(XCeption, self).__init__()
original_model = xception(pretrained="imagenet")
self.features=nn.Sequential(*list(original_model.children())[:-1])
self.last_linear = nn.Sequential(
nn.Linear(original_model.last_linear.in_features, 512),
nn.ReLU(),
nn.Dropout(p=0.5),
nn.Linear(512, num_classes)
)
def logits(self, features):
x = F.relu(features)
x = F.adaptive_max_pool2d(x, (1, 1))
x = x.view(x.size(0), -1)
x = self.last_linear(x)
return x
def forward(self, input):
x = self.features(input)
x = self.logits(x)
return x
device = torch.device("cuda")
model=XCeption(len(class_names))
if torch.cuda.device_count() > 1:
print("Let's use", torch.cuda.device_count(), "GPUs!")
# dim = 0 [30, xxx] -> [10, ...], [10, ...], [10, ...] on 3 GPUs
model = nn.DataParallel(model)
model.to(device)
criterion = nn.CrossEntropyLoss(size_average=False)
optimizer = optim.Adam(model.parameters(), lr=0.0001)
scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, 'min', factor=0.2, patience=5, cooldown=5)
Thank you very much!
Update:
Settings:
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=lr)
scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, 'min', factor=0.2, patience=5, cooldown=5)
model = train_model(model, train_loader, val_loader,
criterion, optimizer, scheduler,
batch_size, trainmult=8, valmult=10,
num_epochs=200, epochs_top=0)
Cleaned training function:
def train_model(model, train_loader, val_loader, criterion, optimizer, scheduler, batch_size, trainmult=1, valmult=1, num_epochs=None, epochs_top=0):
for epoch in range(num_epochs):
for phase in ['train', 'val']:
running_loss = 0.0
running_acc = 0
total = 0
# Iterate over data.
if phase=="train":
model.train(True) # Set model to training mode
for i in range(trainmult):
for data in train_loader:
# get the inputs
inputs, labels = data
inputs, labels = inputs.to(torch.device("cuda")), labels.to(torch.device("cuda"))
# zero the parameter gradients
optimizer.zero_grad()
# forward
outputs = model(inputs) # notinception
_, preds = torch.max(outputs, 1)
loss = criterion(outputs, labels)
# backward + optimize only if in training phase
loss.backward()
optimizer.step()
# statistics
total += labels.size(0)
running_loss += loss.item()*labels.size(0)
running_acc += torch.sum(preds == labels)
train_loss=(running_loss/total)
train_acc=(running_acc.double()/total)
else:
model.train(False) # Set model to evaluate mode
with torch.no_grad():
for i in range(valmult):
for data in val_loader:
# get the inputs
inputs, labels = data
inputs, labels = inputs.to(torch.device("cuda")), labels.to(torch.device("cuda"))
# zero the parameter gradients
optimizer.zero_grad()
# forward
outputs = model(inputs)
_, preds = torch.max(outputs, 1)
loss = criterion(outputs, labels.data)
# statistics
total += labels.size(0)
running_loss += loss.item()*labels.size(0)
running_acc += torch.sum(preds == labels)
val_loss=(running_loss/total)
val_acc=(running_acc.double()/total)
scheduler.step(val_loss)
return model
it may be because type of weight initialization you are using
otherwise this should not happen
try with same initializer in both the models
self.features=nn.Sequential(*list(original_model.children())[:-1])
Are you sure that this line re-instantiates your model in exactly the same way? You're using a NN.Sequential instead of the original XCeption model's forward function. If there's anything in that forward function that isn't the exact same as using a nn.Sequential, it will not reproduce the same performance.
Instead of wrapping it in a Sequential, you could just change this
my_model = Xception()
# load weights before you change the architecture
my_model = load_weights(path_to_weights)
# overwrite the original's last_linear with your own
my_model.last_linear = nn.Sequential(
nn.Linear(original_model.last_linear.in_features, 512),
nn.ReLU(),
nn.Dropout(p=0.5),
nn.Linear(512, num_classes)
)