What does .n method do in this (trainGen.n)(in train_and_evaluate_model() function)
This is the whole code I'm trying to learn
Thank you for your help
def build_data_generators(train_folder, test_folder, labels=None,
image_size=(100, 100), batch_size=50):
train_datagen = ImageDataGenerator(
width_shift_range=0.0,
height_shift_range=0.0,
zoom_range=0.0,
horizontal_flip=True,
vertical_flip=True, # randomly flip images
preprocessing_function=augment_image) # augmentation is done only on
# the train set (and optionally validation)
test_datagen = ImageDataGenerator()
train_gen = train_datagen.flow_from_directory(
train_folder, target_size=image_size, class_mode='sparse',
batch_size=batch_size, shuffle=True, subset='training', classes=labels)
test_gen = test_datagen.flow_from_directory(
test_folder, target_size=image_size, class_mode='sparse',
batch_size=batch_size, shuffle=False, subset=None, classes=labels)
return train_gen, test_gen
def train_and_evaluate_model(model, name="", epochs=2, batch_size=50, verbose=verbose,
useCkpt=False):
print(model.summary())
model_out_dir = os.path.join(output_dir, name)
if not os.path.exists(model_out_dir):
os.makedirs(model_out_dir)
if useCkpt:
model.load_weights(model_out_dir + "/Model.h5")
trainGen, testGen = build_data_generators(
train_dir, test_dir, labels=labels, image_size=image_size, batch_size=batch_size)
optimizer = Adadelta(lr=learning_rate)
model.compile(optimizer=optimizer, loss="sparse_categorical_crossentropy",
metrics=["acc"])
learning_rate_reduction = ReduceLROnPlateau(
monitor='loss', patience=patience, verbose=verbose,
factor=learning_rate_reduction_factor, min_lr=min_learning_rate)
save_model = ModelCheckpoint(
filepath=model_out_dir + "/Model.h5", monitor='loss', verbose=verbose,
save_best_only=True, save_weights_only=False, mode='min', save_freq='epoch')
history = model.fit(trainGen,
epochs=epochs,
steps_per_epoch=(trainGen.n // batch_size) + 1,
verbose=verbose,
callbacks=[learning_rate_reduction, save_model])
model.load_weights(model_out_dir + "/Model.h5")
trainGen.reset()
loss_t, accuracy_t = model.evaluate(trainGen, steps=(trainGen.n // batch_size) + 1,
verbose=verbose)
loss, accuracy = model.evaluate(testGen, steps=(testGen.n // batch_size) + 1,
verbose=verbose)
print("Train: accuracy = %f ; loss_v = %f" % (accuracy_t, loss_t))
print("Test: accuracy = %f ; loss_v = %f" % (accuracy, loss))
plot_model_history(history, out_path=model_out_dir)
testGen.reset()
y_pred = model.predict(testGen, steps=(testGen.n // batch_size) + 1, verbose=verbose)
y_true = testGen.classes[testGen.index_array]
plot_confusion_matrix(y_true, y_pred.argmax(axis=-1), labels, out_path=model_out_dir)
class_report = classification_report(y_true, y_pred.argmax(axis=-1), target_names=labels)
with open(model_out_dir + "/Classification_report.txt", "w") as text_file:
text_file.write("%s" % class_report)
It keeps asking me to put more since I post mostly code so this sentence is just a filler, thank you and my deepest apology for wasting 16.5 seconds of your time to read this whole sentence
These are Keras types. trainGen is a DirectoryIterator which implements Iterator, the class to which this attribute belongs. See the constructor here.
n Integer, total number of samples in the dataset to loop over.
Related
Failed to find data adapter that can handle input: <class 'tensorflow.python.keras.preprocessing.image.ImageDataGenerator'>, <class 'NoneType'>
I am trying to implement deep learning model and I have run the code more than 10 times , some times in 41, 72,88,100 epoch I got this error, is there anybody to help me
def Tuning_Model():
for k in range(FOLDS):
timestamp = datetime.fromtimestamp(time()).strftime('%Y%m%d-%H%M%S')
output_directory = model_path + "\\" + timestamp
if not os.path.exists(output_directory):
os.makedirs(output_directory)
# Training image augmentation
train_data_generator =tf.keras.preprocessing.image.ImageDataGenerator(
rescale=1. / 255,
fill_mode="constant",
shear_range=0.2,
zoom_range=(0.5, 1),
horizontal_flip=True,
rotation_range=360,
channel_shift_range=25,
brightness_range=(0.75, 1.25))
# Validation image augmentation
val_data_generator =tf.keras.preprocessing.image.ImageDataGenerator(
rescale=1. / 255,
fill_mode="constant",
shear_range=0.2,
zoom_range=(0.5, 1),
horizontal_flip=True,
rotation_range=360,
channel_shift_range=25,
brightness_range=(0.75, 1.25))
test_data_generator =tf.keras.preprocessing.image.ImageDataGenerator(rescale=1. / 255)
train_data_generator = train_data_generator.flow_from_directory(
train_data_path,
target_size = RAW_IMG_SIZE,
batch_size = BATCH_SIZE,
class_mode ='categorical'
)
val_data_generator = val_data_generator.flow_from_directory(
val_data_path,
target_size = RAW_IMG_SIZE,
batch_size = BATCH_SIZE,
class_mode = 'categorical')
test_data = test_data_generator.flow_from_directory(
test_data_path,
target_size = IMG_SIZE,
batch_size = BATCH_SIZE,
class_mode = 'categorical',
shuffle = False)
train_data_generator = crop_generator(train_data_generator, IMG_SIZE)
val_data_generator = crop_generator(val_data_generator, IMG_SIZE)
model = Build_Model(model_name)
model_checkpoint = ModelCheckpoint(output_directory + "\\best_model.hdf5", verbose=1, save_best_only=True)
early_stopping = EarlyStopping(patience=STOPPING_PATIENCE, restore_best_weights=True)
reduce_lr = ReduceLROnPlateau('val_loss', factor=0.5, patience=LR_PATIENCE, min_lr=0.000003125)
model.compile(loss='binary_crossentropy', optimizer=Adam(lr=INITIAL_LR), metrics=['categorical_accuracy'])
history = model.fit_generator(
generator = train_data_generator,
steps_per_epoch = train_image_count // BATCH_SIZE,
epochs = epochs,
validation_data = val_data_generator,
validation_steps= val_image_count // BATCH_SIZE,
callbacks = [model_checkpoint, early_stopping, reduce_lr],
shuffle = False)
# Load the last best model
model = load_model(
output_directory + "\\best_model.hdf5")
# Evaluate model on test set
predictions = model.predict_generator(test_data_generator, test_image_count // BATCH_SIZE + 1)
y_true = test_data_generator.classes
y_pred = np.argmax(predictions, axis=1)
print(classification_report(y_true, y_pred, labels=CLASSES, target_names=CLASS_NAMES))
report = classification_report(y_true, y_pred, labels=CLASSES, target_names=CLASS_NAMES, output_dict=True)
with open(output_directory + '\\classification_report.csv', 'w') as f:
for key in report.keys():
f.write("%s,%s\n" % (key, report[key]))
conf_arr = confusion_matrix(y_true, y_pred, labels=CLASSES)
print(conf_arr)
np.savetxt(output_directory + "\\confusion_matrix.csv", conf_arr, delimiter=",")
# Clear model from GPU after each iteration
print("Finished testing fold {}\n".format(k + 1))
K.clear_session()
k = k + 1
if __name__ == '__main__':
Tuning_Model()
ValueError: Failed to find data adapter that can handle input: <class 'tensorflow.python.keras.preprocessing.image.ImageDataGenerator'>, <class 'NoneType
Have you checked whether training/testing data are in the form of numpy arrays before fitting the model?
Also it might be the case where you are importing model from keras like keras.Sequential() but the generator from Tensorflow like tf.keras.preprocessing.image.ImageDataGenerator().
train_generator=datagen.flow_from_dataframe(dataframe=train_df, #directory=data_path,
x_col="Path", y_col="feature_string", seed = 42, classes = chexpert_targets,
class_mode="categorical", target_size=(image_size,image_size), batch_size=32, subset = "training")
validation_generator = datagen.flow_from_dataframe(dataframe=train_df, #directory=data_path,
x_col="Path", y_col="feature_string", seed = 42, classes = chexpert_targets,
class_mode="categorical", target_size=(image_size,image_size), batch_size=16, subset = "validation")
test_generator = test_datagen.flow_from_dataframe(dataframe=valid_only_df, #directory=data_path,
target_size=(image_size,image_size),class_mode='categorical',
batch_size=1, shuffle=False, classes = chexpert_targets,
x_col="Path", y_col="feature_string")
x_col, y_col = next(train_generator)
ds = tf.data.Dataset.from_generator(
lambda: train_generator,
output_types=(tf.float32, tf.float32),
output_shapes=([32,320,320,3], [32,14])
)
I am trying to wrap the image_data_generator.flow_from_dataframe using tf.data but I am finding difficulties, I would really appreciate some help?
I am trying to train a classification model using Keras in a Colab notebook but the problem is that when I try to fit my augmented model it stops in the validation steps showing the error:
ValueError: rate must be a scalar tensor or a float in the range [0, 1), got 1
I have been searching on the web but I can not find this error.The code is the next:
batch_size = 10
img_height= 180
enter code here
img_width = 180
nb_test_samples = 400
nb_val_samples = 200
epochs = 10
datagen = ImageDataGenerator(rescale=1./255)
valid_generator = datagen.flow_from_directory(
'/content/validacion/',
target_size=(img_width, img_height),
batch_size=32,
class_mode='binary')
input_shape = (180,180,3)
model = Sequential()
model.add(Conv2D(32,(3,3),input_shape = input_shape,activation = 'relu'))
model.add(MaxPooling2D(pool_size=((2,2))))
model.add(Conv2D(63,(3,3),activation = 'relu'))
model.add(MaxPooling2D(pool_size=((2,2))))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(64,activation='relu'))
model.add(Dropout(1))
model.add(Dense(1,activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
print(model.summary())
train_datagen_augmented = ImageDataGenerator(
rescale=1./255,
shear_range=0.2,
zoom_range=0.2,
rotation_range = 30,
horizontal_flip=True)
train_generator_augmented = train_datagen_augmented.flow_from_directory(
'/content/entrenamiento',
target_size=(img_width, img_height),
batch_size=batch_size,
class_mode='binary')
history = model.fit_generator(train_generator_augmented,
steps_per_epoch = nb_test_samples // batch_size,
epochs = epochs,
verbose = 1,
validation_data = valid_generator,
*validation_steps = nb_val_samples // batch_size*) #Error here
score = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
I already have the images resized and saved in their respectives directories.
after setting droput rate 0.2, it started working
I am trying to learn how to use multi-models input with flow_from_directory, but there is something I can't figure out. thanks for your help.
The way I understand is that we supply the fit_generator with the two_image_generator and the and the fit method will infer the labels.... what I am missing...
def two_image_generator(generator,
directory,
batch_size,
shuffle = False,
img_size1 = (224,224),
img_size2 = (299,299)):
gen1 = generator.flow_from_directory(
# This is the target directory
directory,
# All images will be resized to target height and width.
target_size=img_size1,
batch_size=batch_size,
# Since we use categorical_crossentropy loss, we need categorical labels
class_mode='categorical',
shuffle = shuffle,
seed = 1)
gen2 = generator.flow_from_directory(
# This is the target directory
directory,
# All images will be resized to target height and width.
target_size=img_size2,
batch_size=batch_size,
# Since we use categorical_crossentropy loss, we need categorical labels
class_mode='categorical',
shuffle = shuffle,
seed = 1)
while True:
X1i = gen1.next()
X2i = gen2.next()
if y_col:
yield [X1i[0], X2i[0]], X1i[1] #X1i[1] is the label
else:
yield [X1i, X2i]
#add data_augmentation
train_aug_datagen = ImageDataGenerator(
rotation_range = 20,
shear_range = 0.1,
zoom_range = 0.2,
width_shift_range = 0.1,
height_shift_range = 0.1,
horizontal_flip = True
)
train_generator = two_image_generator(train_aug_datagen,
train_dir,
batch_size = batch_size,
shuffle = True)
validation_datagen = ImageDataGenerator()
validation_generator = two_image_generator(validation_datagen,
validation_dir,
batch_size = batch_size,
shuffle = True)
def create_base_model(MODEL, img_size, lambda_fun = None):
inp = Input(shape = (img_size[0], img_size[1], 3))
x = inp
if lambda_fun:
x = Lambda(lambda_fun)(x)
base_model = MODEL(input_tensor = x, weights = 'imagenet',
include_top = False, pooling = 'avg')
model = Model(inp, base_model.output)
return model
#define vgg + resnet50 + densenet
model1 = create_base_model(vgg16.VGG16, (224, 224), vgg16.preprocess_input)
model2 = create_base_model(resnet50.ResNet50, (224, 224), resnet50.preprocess_input)
model3 = create_base_model(inception_v3.InceptionV3, (299, 299), inception_v3.preprocess_input)
model1.trainable = False
model2.trainable = False
model3.trainable = False
inpA = Input(shape = (224, 224, 3))
inpB = Input(shape = (299, 299, 3))
out1 = model1(inpA)
out2 = model2(inpA)
out3 = model3(inpB)
x = Concatenate()([out1, out2, out3])
x = Dropout(0.2)(x)
x = Dense(2, activation='softmax')(x)
model = Model([inpA, inpB], x)
############################################################################
trained_models_path = './models/VggFace_best_model'
model_names = trained_models_path + '_epoch_{epoch:02d}_val_acc_{val_accuracy:.4f}.hdf5'
checkpoint = ModelCheckpoint(model_names, 'val_accuracy', verbose=1, save_best_only=True)
############################################################################
early = EarlyStopping(monitor='val_loss', min_delta=0, patience=3, verbose=1, mode='auto')
callbacks = [checkpoint,early]
history = model.fit_generator(train_generator,
steps_per_epoch= NUM_TRAIN //batch_size,
epochs=100,
validation_data=validation_generator,
validation_steps= NUM_TEST //batch_size,
verbose=1,
use_multiprocessing=True,
workers=14,
callbacks=callbacks )
NameError: name 'y_col' is not defined
#add data_augmentation
train_aug_datagen = ImageDataGenerator(
rescale = 1./255,
rotation_range = 20,
shear_range = 0.1,
zoom_range = 0.2,
width_shift_range = 0.1,
height_shift_range = 0.1,
horizontal_flip = True
)
validation_datagen = ImageDataGenerator(rescale = 1./255)
def two_image_generator(generator,
directory,
batch_size,
shuffle = False,
img_size1 = (224,224),
img_size2 = (299,299)):
gen1 = generator.flow_from_directory(
# This is the target directory
directory,
# All images will be resized to target height and width.
target_size=img_size1,
batch_size=batch_size,
# Since we use categorical_crossentropy loss, we need categorical labels
class_mode='categorical',
shuffle = shuffle,
seed = 7)
gen2 = generator.flow_from_directory(
# This is the target directory
directory,
# All images will be resized to target height and width.
target_size=img_size2,
batch_size=batch_size,
# Since we use categorical_crossentropy loss, we need categorical labels
class_mode='categorical',
shuffle = shuffle,
seed = 7)
while True:
X1i = gen1.next()
X2i = gen2.next()
yield [X1i[0], X2i[0]], X2i[1] #Yield both images and their mutual label
train_generator = two_image_generator(train_aug_datagen,
train_dir,
batch_size = batch_size,
shuffle = True)
validation_generator = two_image_generator(validation_datagen,
validation_dir,
batch_size = batch_size,
shuffle = True)
############################################################################
trained_models_path = './models/VggFace_best_model'
model_names = trained_models_path + '_epoch_{epoch:02d}_val_acc_{val_accuracy:.4f}.hdf5'
checkpoint = ModelCheckpoint(model_names, 'val_accuracy', verbose=1, save_best_only=True)
############################################################################
early = EarlyStopping(monitor='val_loss', min_delta=0, patience=3, verbose=1, mode='auto')
callbacks = [checkpoint,early]
history = model.fit_generator(train_generator,
steps_per_epoch= NUM_TRAIN //batch_size,
epochs=100,
validation_data=validation_generator,
validation_steps= NUM_TEST //batch_size,
verbose=1,
use_multiprocessing=True,
# workers=14,
callbacks=callbacks )
Epoch 1/100
Found 5000 images belonging to 2 classes.
Found 5000 images belonging to 2 classes.
Found 52700 images belonging to 2 classes.
Found 52700 images belonging to 2 classes.
340/625 [===============>..............] - ETA: 4:37 - loss: 7.7634 - acc: 0.4926
import pandas as pd
I'm new to Keras and I'm trying to train a face detection machine in Python. As you can see the generator returned values but it seems that the output is not in suitable format. Any advice is highly appreciated
The full ValueError is as following:
ValueError: Output of generator should be a tuple (x, y, sample_weight)
or (x, y). Found: [[[[0.10196079 0.08235294 0.07058824]
[0.10196079 0.08235294 0.07058824]
[0.10196079 0.08235294 0.07058824]
...
[0.10196079 0.08235294 0.07058824]
[0.10196079 0.08235294 0.07058824]
[0.10196079 0.08235294 0.07058824]]
Here is the traceback
File "C:/Users/user/PycharmProjects/untitled4/transferLearning.py", line > 103, in callbacks=[checkpoint, early])
File "C:\Users\user\Anaconda3\lib\site->packages\keras\legacy\interfaces.py", line 91, in wrapper
return func(*args, **kwargs)
File "C:\Users\user\Anaconda3\lib\site-packages\keras\engine\training.py", line 1418, in fit_generator
initial_epoch=initial_epoch)
File "C:\Users\user\Anaconda3\lib\site->packages\keras\engine\training_generator.py", line 198, in fit_generator
str(generator_output))
Full code below
image_dir = path.join(root_dir, 'train_countinghead', 'image_data')
img_width, img_height = 256, 256
train_csv = pandas.read_csv(path.join(root_dir, 'train_countinghead', 'train.csv'))
test_csv = pandas.read_csv(path.join(root_dir, 'test_headcount.csv'))
train_samples = len(train_csv)
test_samples = len(test_csv)
batch_size = 16
epochs = 50
model = applications.VGG16(weights='imagenet', include_top=False, input_shape=(img_width, img_height, 3))
# Freeze the layers which you don't want to train. Here I am freezing the first 5 layers.
for layer in model.layers[:5]:
layer.trainable = False
# Adding custom Layers
x = model.output
x = Flatten()(x)
x = Dense(1024, activation="relu")(x)
x = Dropout(0.5)(x)
x = Dense(1024, activation="relu")(x)
predictions = Dense(16, activation="softmax")(x)
# creating the final model
model_final = Model(inputs=model.input, outputs=predictions)
# compile the model
model_final.compile(loss="categorical_crossentropy", optimizer=optimizers.SGD(lr=0.0001, momentum=0.9),
metrics=["accuracy"])
# Initiate the train and test generators with data Augumentation
train_datagen = ImageDataGenerator(
rescale=1./255,
horizontal_flip=True,
fill_mode="nearest",
zoom_range=0.3,
width_shift_range=0.3,
height_shift_range=0.3,
rotation_range=30
)
test_datagen = ImageDataGenerator(
rescale=1. / 255,
horizontal_flip=True,
fill_mode="nearest",
zoom_range=0.3,
width_shift_range=0.3,
height_shift_range=0.3,
rotation_range=30
)
# if `class_mode` is `"categorical"` (default value) it must include the `y_col` column with the class/es of each image.
# Check the comments in method definition for more
train_generator = train_datagen.flow_from_dataframe(
dataframe=train_csv,
directory=image_dir,
x_col='Name',
target_size=(img_height, img_width),
batch_size=batch_size,
class_mode=None
)
test_generator = test_datagen.flow_from_dataframe(
dataframe=test_csv,
directory=image_dir,
x_col='Name',
target_size=(img_height, img_width),
batch_size=batch_size,
class_mode=None
)
# Save the model according to the conditions
checkpoint = ModelCheckpoint(path.join(root_dir, "vgg16_1.h5"), monitor='val_acc', verbose=1, save_best_only=True,
save_weights_only=False,
mode='auto', period=1)
early = EarlyStopping(monitor='val_acc', min_delta=0, patience=10, verbose=1, mode='auto')
# Train the model
model_final.fit_generator(
train_generator,
# samples_per_epoch=train_samples,
steps_per_epoch=train_samples / batch_size,
epochs=epochs,
validation_data=test_generator,
validation_steps=test_samples / batch_size,
callbacks=[checkpoint, early])
The problem is that here you didn't provide a target column.
If you look at the documentation, you can see that you need (because you are training your model) to specify a y_col and also not have class_mode=None (which is only used for prediction), at least for the train_generator (I don't know what you plan to do with the test_generator).
You could also have seen that using the error, which was telling you that it wasn't getting all the necessary elements (x the data, y the label).