I have some code for a mixed model, one that trains on an efficient net and the rest on some external data that I have combined. The following is an example for the model:
def create_model():
# Define parameters
inputShape = (256,256,3)
inputDim = 8
# define MLP network
model = Sequential()
model.add(Dense(8, input_dim=inputDim, activation="relu"))
model.add(Dense(4, activation="relu"))
cnnModel = Sequential()
cnnModel.add(EfficientNetB5(include_top = False, input_shape=inputShape))
cnnModel.add(Flatten())
cnnModel.add(Dense(units = 16, activation='relu'))
cnnModel.add(Dense(units = 4, activation='relu'))
# Concatenate them
fullModel = concatenate([cnnModel.output,model.output])
fullModel = Dense(4, activation="relu")(fullModel)
fullModel = Dense(1, activation="sigmoid")(fullModel)
model = Model(inputs=[cnnModel.input,model.input], outputs=fullModel)
return model
However, when I run this through the fit_generator function I recieve the following error:
batch_size = 16
train_steps = TrainData.shape[0]//batch_size
valid_steps = TrainData.shape[0]//batch_size
model = create_model()
opt = Adam(lr=1e-3, decay=1e-3 / 200)
model.compile(loss="binary_crossentropy", optimizer=opt)
print("[INFO] training model...")
model.fit_generator(
train_dl,
epochs=3,
steps_per_epoch = train_steps
)
model.save("models/final_model")
InvalidArgumentError: Incompatible shapes: [16,3,256,256] vs. [1,1,1,3]
[[node model_47/efficientnetb5/normalization_52/sub (defined at <ipython-input-262-76be6a4af4a4>:11) ]] [Op:__inference_train_function_1072272]
I'm unsure where this error is coming from, either in the data loader or in the efficient net. Any ideas?
Edit to include data loader:
def data_generator(image_dir, dataframe, min_max, binary, category, transforms = None, batch_size = 16):
i = 0
samples_per_epoch = dataframe.shape[0]
number_of_batches = samples_per_epoch/batch_size
while True:
batch = {'images': [], 'data': [], 'labels': []} # use a dict for multiple inputs
# Randomly sample images in dataframe
idx = i
img_path = f"{image_dir}/{dataframe.iloc[idx]['image_name']}.jpg"
img = Image.open(img_path)
if transforms:
img = transforms(**{"image": np.array(img)})["image"]
img = np.asarray( img, dtype="int32" )
# make data into tensors
dataframe2 = dataframe.iloc[idx]
data_cont = min_max.transform(np.array(dataframe2['age_approx']).reshape(1, -1))
data_bina = binary.transform(dataframe2['sex'])
data_cate = category.transform(dataframe2['anatom_site_general_challenge'])
data_total = np.concatenate((data_cont, data_bina, data_cate), axis = 1)
label = dataframe2['target']
batch['images'].append(img)
batch['data'].append(data_total)
batch['labels'].append(label)
batch['images'] = np.array(batch['images']) # convert each list to array
batch['data'] = np.array(batch_x['data'])
batch['labels'] = np.array(batch['labels'])
i += 1
if counter >= number_of_batches:
counter = 0
yield [batch['images'], batch['data']], batch['labels']
def get_data(train_df, valid_df, train_tfms, test_tfms, batch_size, min_max, binary, category):
train_dl = data_generator(image_dir='train/', dataframe = train_df, batch_size = batch_size, min_max = min_max, binary = binary, category = category, transforms = train_tfms)
valid_dl = data_generator(image_dir='train/', dataframe = valid_df, batch_size = batch_size*2, min_max = min_max, binary = binary, category = category, transforms = test_tfms)
return train_dl, valid_dl
I seem to have the same issue when I just used the images and the efficient net. It seems like using the Keras inbuilt image data loader functions is the only way I can get it to work (with just images).
Related
I am trying to do a deep learning model using Keras and VGG16 algorithm on DICOM images.
I used thhe following data generator to process my images:
``
# tested on tf 2.1
from keras_preprocessing.image.dataframe_iterator import DataFrameIterator
class DCMDataFrameIterator(DataFrameIterator):
def __init__(self, *arg, **kwargs):
self.white_list_formats = ('dcm')
super(DCMDataFrameIterator, self).__init__(*arg, **kwargs)
self.dataframe = kwargs['dataframe']
self.x = self.dataframe[kwargs['x_col']]
self.y = self.dataframe[kwargs['y_col']]
self.color_mode = kwargs['color_mode']
self.target_size = kwargs['target_size']
def _get_batches_of_transformed_samples(self, indices_array):
# get batch of images
batch_x = np.array([self.read_dcm_as_array(dcm_path, self.target_size, color_mode=self.color_mode)
for dcm_path in self.x.iloc[indices_array]])
batch_y = np.array(self.y.iloc[indices_array].astype(np.uint8)) # astype because y was passed as str
# transform images
if self.image_data_generator is not None:
for i, (x, y) in enumerate(zip(batch_x, batch_y)):
transform_params = self.image_data_generator.get_random_transform(x.shape)
batch_x[i] = self.image_data_generator.apply_transform(x, transform_params)
# you can change y here as well, eg: in semantic segmentation you want to transform masks as well
# using the same image_data_generator transformations.
return batch_x, batch_y
#staticmethod
def read_dcm_as_array(dcm_path, target_size=(256, 256), color_mode='rgb'):
img = tf.io.read_file(dcm_path)
img = tfio.image.decode_dicom_image(img, dtype=tf.uint16)
img = tf.image.resize(img, target_size)
img = tf.image.grayscale_to_rgb(img, name=None) # convert image grayscale to rgb for model VG16
#img = np.expand_dims(img, -1)
return img
``
and:
``
# you can use preprocessing_function instead of rescale in all generators
# if you are using a pretrained network
train_augmentation_parameters = dict(
rescale=1.0/255.0,
rotation_range=10,
zoom_range=0.2,
horizontal_flip=True,
fill_mode='nearest',
brightness_range = [0.8, 1.2],
validation_split = 0.2
)
valid_augmentation_parameters = dict(
rescale=1.0/255.0,
validation_split = 0.2
)
test_augmentation_parameters = dict(
rescale=1.0/255.0
)
# training parameters
BATCH_SIZE = 32
CLASS_MODE = 'sparse'
COLOR_MODE = 'grayscale'
TARGET_SIZE = (300, 300)
EPOCHS = 10
SEED = 1337
train_consts = {
'seed': SEED,
'batch_size': BATCH_SIZE,
'class_mode': CLASS_MODE,
'color_mode': COLOR_MODE,
'target_size': TARGET_SIZE,
'subset': 'training'
}
valid_consts = {
'seed': SEED,
'batch_size': BATCH_SIZE,
'class_mode': CLASS_MODE,
'color_mode': COLOR_MODE,
'target_size': TARGET_SIZE,
'subset': 'validation'
}
test_consts = {
'batch_size': 1, # should be 1 in testing
'class_mode': CLASS_MODE,
'color_mode': COLOR_MODE,
'target_size': TARGET_SIZE, # resize input images
'shuffle': False
}
``
and:
``
# Using the training phase generators
train_augmenter = ImageDataGenerator(**train_augmentation_parameters)
valid_augmenter = ImageDataGenerator(**valid_augmentation_parameters)
train_generator = DCMDataFrameIterator(dataframe=df_merged,
x_col='files',
y_col='class',
image_data_generator=train_augmenter,
**train_consts)
valid_generator = DCMDataFrameIterator(dataframe=df_merged,
x_col='files',
y_col='class',
image_data_generator=valid_augmenter,
**valid_consts)
``
Result of last command:
``
Found 7828 validated image filenames belonging to 4 classes.
Found 1956 validated image filenames belonging to 4 classes.
``
After:
``
base_model = VGG16(weights='imagenet', include_top=False)
n_class = 4 #
# Freezer les couches du VGG16
for layer in base_model.layers:
layer.trainable = False
model = Sequential()
model.add(base_model) # Ajout du modèle VGG16
model.add(GlobalAveragePooling2D())
model.add(Dense(1024,activation='relu'))
model.add(Dropout(rate=0.2))
model.add(Dense(512, activation='relu'))
model.add(Dropout(rate=0.2))
model.add(Dense(n_class, activation='softmax'))
``
I compile but but when I do .fit I have the following message:
ValueError: Input arrays must be multi-channel 2D images.
Could you help me ?
Thank you in advance,
Thibaut
I had a look on stackoverflow to try to find a solution !
I'm having troubles making the Preprocessing layers and the Keras Tuner cooperate.
I am referring to this tutorial Load CSV data | TensorFlow Core for the Preprocessing part, and to the Getting started with KerasTuner documentation for the Keras Tuner part.
Briefly, here's the code.
He loads the data:
titanic = pd.read_csv("https://storage.googleapis.com/tf-datasets/titanic/train.csv")
titanic_features = titanic.copy()
titanic_labels = titanic_features.pop('survived')
He creates the symbolic tensors of the features in a dictionary
inputs = {}
for name, column in titanic_features.items():
dtype = column.dtype
if dtype == object:
dtype = tf.string
else:
dtype = tf.float32
inputs[name] = tf.keras.Input(shape=(1,), name=name, dtype=dtype)
inputs
Then he applies normalization on the numerical features:
numeric_inputs = {name:input for name,input in inputs.items()
if input.dtype==tf.float32}
x = layers.Concatenate()(list(numeric_inputs.values()))
norm = layers.Normalization()
norm.adapt(np.array(titanic[numeric_inputs.keys()]))
all_numeric_inputs = norm(x)
all_numeric_inputs
He creates a list
preprocessed_inputs = [all_numeric_inputs]
He one hot encodes the categorical features:
for name, input in inputs.items():
if input.dtype == tf.float32:
continue
lookup = layers.StringLookup(vocabulary=np.unique(titanic_features[name]))
one_hot = layers.CategoryEncoding(num_tokens=lookup.vocabulary_size())
x = lookup(input)
x = one_hot(x)
preprocessed_inputs.append(x)
and then He concatenates it:
preprocessed_inputs_cat = layers.Concatenate()(preprocessed_inputs)
titanic_preprocessing = tf.keras.Model(inputs, preprocessed_inputs_cat)
Now what I want to do is insert this preprocessing part in KerasTuner. I tried this:
def titanic_model(units, activation):
model_inputs = tf.keras.Input(shape=28)
dense_1 = layers.Dense(units=units, activation=activation)(model_inputs)
dense_output = layers.Dense(1)(dense_1)
body = tf.keras.Model(inputs = model_inputs, outputs = dense_output)
return body
def build_model(hp,preprocessing_head, inputs):
units = hp.Int("units", min_value=32, max_value=512, step=32)
activation = hp.Choice("activation", ["relu", "tanh"])
preprocessed_inputs = preprocessing_head(inputs)
result = titanic_model(units,
activation)(preprocessed_inputs)
model = tf.keras.Model(inputs, result)
model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
optimizer=tf.keras.optimizers.Adam(),
metrics = ['accuracy'])
return model
titanic_model = build_model(keras_tuner.HyperParameters(),titanic_preprocessing, inputs)
but it gives me the following error:
Inputs to a layer should be tensors. Got: <keras_tuner.engine.hyperparameters.HyperParameters
object at 0x7ff52844da30>
I cannot understand if I am close to the solution, or this is not the right way to proceed.
However, the workaround i found was to insert directly in the build_model function the preprocessing layer (titanic_preprocessing) and the inputs dictionary, without passing it as an argument of the function.
Hence:
def titanic_model(units, activation):
model_inputs = tf.keras.Input(shape=28)
dense_1 = layers.Dense(units=units, activation=activation)(model_inputs)
dense_output = layers.Dense(1)(dense_1)
body = tf.keras.Model(inputs = model_inputs, outputs = dense_output)
return body
def build_model(hp):
units = hp.Int("units", min_value=32, max_value=512, step=32)
activation = hp.Choice("activation", ["relu", "tanh"])
preprocessed_inputs = titanic_preprocessing(inputs)
result = titanic_model(units,
activation)(preprocessed_inputs)
model = tf.keras.Model(inputs, result)
model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
optimizer=tf.keras.optimizers.Adam(),
metrics = ['accuracy'])
return model
titanic_model = build_model()
In this case it seems to work, and by setting the tuner
tuner = keras_tuner.RandomSearch(
hypermodel = build_model,
objective=keras_tuner.Objective("accuracy", direction="max"),
max_trials = 1,
overwrite = True,
directory = "tuner_dir",
project_name = "regression_tuner")
and searching it works:
tuner.search(x=titanic_features_dict, y=titanic_labels, epochs=10)
However, I have doubts about this solution and would appreciate feedback on this.
Thank you!
I'm trying to figure out what is the difference between using a pretrained model from tensorflow hub versus using the very same architecture from tf.keras.applications. I've tried training 2 models with the same architecture - one from tf hub, an the other one from tf.keras.applications, which should yeild comparable results, however the results are vastly different. Could you please explain the difference?
Here are examples of the two models.
base_model = tf.keras.applications.EfficientNetB0(include_top = False)
base_model.trainable = False
inputs = Input(shape = (224,224,3), name = 'input_layer')
x = base_model(inputs, training = False)
x = GlobalAveragePooling2D(name = 'global_avg_pool_layer')(x)
outputs = Dense(len(class_names), activation = 'softmax', name = 'output_layer')(x)
model_1 = tf.keras.Model(inputs, outputs)
model_1.compile(loss = 'categorical_crossentropy', optimizer = Adam(), metrics = ['accuracy'])
history_1 = model_1.fit(train_data_all_10_percent,
epochs = 10,
validation_data = test_data,
validation_steps = (0.15 * len(test_data)))
AND THE SECOND
efficientnet_url = 'https://tfhub.dev/tensorflow/efficientnet/b0/feature-vector/1'
def create_model(model_url, num_classes = 10):
feature_extractor_layer = hub.KerasLayer(model_url, trainable = False, name = 'feature_extraction_layer', input_shape = IMG_SIZE + (3,))
model = Sequential([
feature_extractor_layer,
Dense(len(class_names), activation = 'softmax', name = 'output_layer')
])
return model
efficientnet_model = create_model(efficientnet_model , num_classes = len(class_names))
efficientnet_model .compile(loss = 'categorical_crossentropy', optimizer = Adam(), metrics = ['accuracy'])
efficientnet_history = efficientnet_model .fit(train_data_all_10_percent,
epochs = 10,
validation_data = test_data,
validation_steps = 0.15 * len(test_data))
I am building a small network using some custom network boxes for each use case, It looks like this :
def top_block(dropout = None, training = None):
# scaled input
input_1 = tf.keras.Input(shape=(1,15), dtype='float32')
input_2 = tf.keras.Input(shape=(1,15), dtype='float32')
if dropout:
layer_one = tf.keras.layers.Dropout(rate = dropout)(input_1, training = training)
layer_two = tf.keras.layers.Dropout(rate = dropout)(input_2, training = training)
return [layer_one,layer_two]
return [input_1,input_2]
def bottom_layer(input_layers):
data = tf.reduce_mean(input_layers,0)
cls_layer = tf.keras.layers.Dense(1,
kernel_initializer = keras.initializers.glorot_uniform(seed=200),
activation = 'sigmoid')(data)
model = tf.keras.Model([input_layers[0], input_layers[1]], cls_layer , name = 'model_1')
model.compile(loss = 'binary_crossentropy', optimizer = 'adam', metrics=['accuracy'])
model.summary()
return model
If I am trying to access this network without dropout, it's working fine :
top_ = top_block()
model = bottom_layer(top_ )
But if I am accessing with dropout, it's giving error:
top_ = top_block(dropout = 0.2, training = True)
model = bottom_layer(top_ )
ValueError: Graph disconnected: cannot obtain value for tensor Tensor("input_72:0", shape=(None, 1, 15), dtype=float32) at layer "input_72". The following previous layers were accessed without issue: []
How to access the model with dropout layer?
How to disable training = False during evaluate? Do I need to load full model and old model weights?
Thank You!
I just realized my input is coming from intermediate layer (dropout layer), It should come directly from Input layer :
def top_block():
# scaled input
input_1 = tf.keras.Input(shape=(1,15), dtype='float32')
input_2 = tf.keras.Input(shape=(1,15), dtype='float32')
return [input_1, input_2]
def apply_dropout(layers_data, dropout_val, training):
layer_one = tf.keras.layers.Dropout(rate = dropout_val)(layers_data[0], training = training)
layer_two = tf.keras.layers.Dropout(rate = dropout_val)(layers_data[1], training = training)
return [layer_one, layer_two]
def bottom_layer(input_layers, data):
data = tf.reduce_mean(data, 0)
cls_layer = tf.keras.layers.Dense(1,
kernel_initializer = keras.initializers.glorot_uniform(seed=200),
activation = 'sigmoid')(data)
model = tf.keras.Model(input_layers, cls_layer , name = 'model_1')
model.compile(loss = 'binary_crossentropy', optimizer = 'adam', metrics=['accuracy'])
model.summary()
return model
It's working now
top_ = top_block()
dropout_ = apply_dropout(top_, 0.2, True)
model = bottom_layer(top_ , dropout_)
I would like to write a custom loss function in Keras that takes in a tensor of predicted values and then for each process it through another function before adding to the loss. Here is what this looks like. This is a complete code example, please look at the function sample_loss.
class AE():
def __init__(self,inputSize=78,numNeurons = 50, latentSize=5,expSize = 10,batchSize = 1, activation="relu"):
K.clear_session()
self.inputSize = inputSize
self.latentSize = latentSize
self.activation = activation
self.numNeurons = numNeurons
self.expSize = expSize
self.batchSize = batchSize
self.encoder = self.createEncoder()
self.decoder = self.createDecoder()
self.filterModel = self.createFilterModel()
self.AE = self.createAE()
def sample_loss(self,y_pred ):
loss = 0.0
for j in tf.range(0,self.inputSize ,2):
pt = y_pred[j:j+2]
loss += K.sum(self.filterModel.predict(pt))
return loss
def createEncoder(self):
# create the encoder
xIn = Input(shape=(self.inputSize,), name="data_in")
y_train = Input(shape=(self.latentSize,), name='y_train')
x = Dense(self.numNeurons,activation=self.activation)(xIn)
xlatent = Dense(self.latentSize,activation=self.activation)(x)
# create the encoder
encoder = Model(xIn,xlatent)
return encoder
def createDecoder(self):
# create the decoder
latentIn = Input(shape=(self.latentSize,))
x = Dense(self.numNeurons,activation=self.activation)(latentIn)
out = Dense(self.inputSize,activation="linear")(x)
# create a decoder
decoder = Model(latentIn,out)
return decoder
def createAE(self):
xIn = Input(shape=(self.inputSize,), name="ae_data_in")
# create the total model
latentOutput = self.encoder(xIn)
dataOutput = self.decoder(latentOutput)
model = Model(inputs=xIn,outputs=dataOutput)
model.add_loss( self.sample_loss( dataOutput) )
model.summary()
return model
def createFilterModel(self):
xIn = Input(shape=(2,), name="filter_data_in")
x = Dense(4, activation='sigmoid')(xIn)
x = Dense(1, activation='sigmoid')(x)
model = Model(xIn,x)
model.compile(optimizer='adam', loss='binary_crossentropy')
return model
modelAE = AE()
modelAE.AE.compile(optimizer='adam',loss="mse", metrics=['accuracy'])
So I have a dictionary of models, filterModels. These are actually predictive models in Keras. For each model in this dictionary, I want to pass it the corresponding part of y_pred and then add its output to the loss.
How can I do this?
Here is the error the current code gives me:
'''
ValueError: When feeding symbolic tensors to a model, we expect the tensors to have a static batch size. Got tensor with shape: (None, 78)
'''