I am using the following, fairly simple code to predict an output variable which may have 3 categories:
n_factors = 20
np.random.seed = 42
def embedding_input(name, n_in, n_out, reg):
inp = Input(shape=(1,), dtype='int64', name=name)
return inp, Embedding(n_in, n_out, input_length=1, W_regularizer=l2(reg))(inp)
user_in, u = embedding_input('user_in', n_users, n_factors, 1e-4)
artifact_in, a = embedding_input('artifact_in', n_artifacts, n_factors, 1e-4)
mt = Input(shape=(31,))
mr = Input(shape=(1,))
sub = Input(shape=(24,))
def onehot(featurename):
onehot_encoder = OneHotEncoder(sparse=False)
onehot_encoded = onehot_encoder.fit_transform(Modality_Durations[featurename].reshape(-1, 1))
trn_onehot_encoded = onehot_encoded[msk]
val_onehot_encoded = onehot_encoded[~msk]
return trn_onehot_encoded, val_onehot_encoded
trn_onehot_encoded_mt, val_onehot_encoded_mt = onehot('modality_type')
trn_onehot_encoded_mr, val_onehot_encoded_mr = onehot('roleid')
trn_onehot_encoded_sub, val_onehot_encoded_sub = onehot('subject')
trn_onehot_encoded_quartile, val_onehot_encoded_quartile = onehot('quartile')
# Model
x = merge([u, a], mode='concat')
x = Flatten()(x)
x = merge([x, mt], mode='concat')
x = merge([x, mr], mode='concat')
x = merge([x, sub], mode='concat')
x = Dense(10, activation='relu')(x)
BatchNormalization()
x = Dense(3, activation='softmax')(x)
nn = Model([user_in, artifact_in, mt, mr, sub], x)
nn.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
nn.optimizer.lr = 0.001
nn.fit([trn.member_id, trn.artifact_id, trn_onehot_encoded_mt, trn_onehot_encoded_mr, trn_onehot_encoded_sub], trn_onehot_encoded_quartile,
batch_size=256,
epochs=2,
validation_data=([val.member_id, val.artifact_id, val_onehot_encoded_mt, val_onehot_encoded_mr, val_onehot_encoded_sub], val_onehot_encoded_quartile)
)
Here's the summary of the model:
____________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
====================================================================================================
user_in (InputLayer) (None, 1) 0
____________________________________________________________________________________________________
artifact_in (InputLayer) (None, 1) 0
____________________________________________________________________________________________________
embedding_9 (Embedding) (None, 1, 20) 5902380 user_in[0][0]
____________________________________________________________________________________________________
embedding_10 (Embedding) (None, 1, 20) 594200 artifact_in[0][0]
____________________________________________________________________________________________________
merge_25 (Merge) (None, 1, 40) 0 embedding_9[0][0]
embedding_10[0][0]
____________________________________________________________________________________________________
flatten_7 (Flatten) (None, 40) 0 merge_25[0][0]
____________________________________________________________________________________________________
input_13 (InputLayer) (None, 31) 0
____________________________________________________________________________________________________
merge_26 (Merge) (None, 71) 0 flatten_7[0][0]
input_13[0][0]
____________________________________________________________________________________________________
input_14 (InputLayer) (None, 1) 0
____________________________________________________________________________________________________
merge_27 (Merge) (None, 72) 0 merge_26[0][0]
input_14[0][0]
____________________________________________________________________________________________________
input_15 (InputLayer) (None, 24) 0
____________________________________________________________________________________________________
merge_28 (Merge) (None, 96) 0 merge_27[0][0]
input_15[0][0]
____________________________________________________________________________________________________
dense_13 (Dense) (None, 10) 970 merge_28[0][0]
____________________________________________________________________________________________________
dense_14 (Dense) (None, 3) 33 dense_13[0][0]
====================================================================================================
Total params: 6,497,583
Trainable params: 6,497,583
Non-trainable params: 0
_____________________________
But on the fit statement, I get the following error:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-71-7de0782d7d5d> in <module>()
5 batch_size=256,
6 epochs=2,
----> 7 validation_data=([val.member_id, val.artifact_id, val_onehot_encoded_mt, val_onehot_encoded_mr, val_onehot_encoded_sub], val_onehot_encoded_quartile)
8 )
9 # nn.fit([trn.member_id, trn.artifact_id, trn_onehot_encoded_mt, trn_onehot_encoded_mr, trn_onehot_encoded_sub], trn.duration_new,
/home/prateek_dl/anaconda3/lib/python3.5/site-packages/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)
1520 class_weight=class_weight,
1521 check_batch_axis=False,
-> 1522 batch_size=batch_size)
1523 # Prepare validation data.
1524 do_validation = False
/home/prateek_dl/anaconda3/lib/python3.5/site-packages/keras/engine/training.py in _standardize_user_data(self, x, y, sample_weight, class_weight, check_batch_axis, batch_size)
1380 output_shapes,
1381 check_batch_axis=False,
-> 1382 exception_prefix='target')
1383 sample_weights = _standardize_sample_weights(sample_weight,
1384 self._feed_output_names)
/home/prateek_dl/anaconda3/lib/python3.5/site-packages/keras/engine/training.py in _standardize_input_data(data, names, shapes, check_batch_axis, exception_prefix)
142 ' to have shape ' + str(shapes[i]) +
143 ' but got array with shape ' +
--> 144 str(array.shape))
145 return arrays
146
ValueError: Error when checking target: expected dense_14 to have shape (None, 1) but got array with shape (1956554, 3)
How do I resolve this error? Why is the final layer expecting (None,1) when according to the summary() it has to output (None,3)?
Any help would be greatly appreciated.
I fixed the error using categorical_entropy instead of sparse_categorical_entropy.
Related
can anyone help me ..where is the error here?? can anyone help me please. i am trying to train a model for image captioning in persian language.i try to build my model .. below is the code and the model summary.....................
`embeddings_dim = 256
input_image_dim = X_train_image.shape[1]
keras.backend.clear_session()
# image model
input_image = keras.layers.Input(shape=(input_image_dim,), name='input_image')
#input_image_dropout = keras.layers.Dropout(0.4)(input_image)
image_embeddings = keras.layers.Dense(embeddings_dim, activation='tanh', name='image_embeddings') (input_image)
# text model
# Set up the decoder, using `image_embeddings` as initial state.
decoder_inputs = keras.layers.Input(shape=(max_len,))
embeddings = keras.layers.Embedding(len(word2index), embeddings_dim, mask_zero=True)(decoder_inputs)
#embeddings_dropout = keras.layers.Dropout(0.3)(embeddings)
gru = keras.layers.GRU(embeddings_dim)(embeddings, initial_state=image_embeddings) # , return_sequences=True
#flat = keras.layers.Flatten()(gru)
dense = keras.layers.Dense(embeddings_dim, activation='relu')(gru)
#dense_dropout = keras.layers.Dropout(0.3)(dense)
decoder_outputs = keras.layers.Dense(len(word2index), activation='softmax')(dense)
seq2seq = keras.Model([input_image, decoder_inputs], decoder_outputs)
seq2seq.summary()
# prepare callback
#histories = My_Callback()
model_checkpoint_path = 'models/model.{epoch:02d}-{val_loss:.3f}--{b1:.3f}.hdf5'
checkpoint_callback = keras.callbacks.ModelCheckpoint(model_checkpoint_path, monitor='val_loss', verbose=1, save_best_only=False, save_weights_only=False, mode='auto', period=1)
callbacks = [TQDMNotebookCallback(), Bleu_Callback(), checkpoint_callback] #[checkpoint_callback, TQDMNotebookCallback(), My_Callback()]
seq2seq.compile(optimizer=keras.optimizers.Adam(), loss='categorical_crossentropy')
seq2seq.fit([X_train_image, X_train_text], y_train_text,
validation_data=([X_test_image, X_test_text], y_test_text),
batch_size=1024,
epochs=10,
verbose=2,
callbacks=callbacks) # add My_Callback() to callbacks to calculate & display BLEU score after each epoch`
this is the error after running my code .. gives me KeyError: 'metrics'... thanks
` Model: "model"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
input_1 (InputLayer) [(None, 20)] 0 []
input_image (InputLayer) [(None, 1000)] 0 []
embedding (Embedding) (None, 20, 256) 730112 ['input_1[0][0]']
image_embeddings (Dense) (None, 256) 256256 ['input_image[0][0]']
gru (GRU) (None, 256) 394752 ['embedding[0][0]',
'image_embeddings[0][0]']
dense (Dense) (None, 256) 65792 ['gru[0][0]']
dense_1 (Dense) (None, 2852) 732964 ['dense[0][0]']
==================================================================================================
Total params: 2,179,876
Trainable params: 2,179,876
Non-trainable params: 0
__________________________________________________________________________________________________
WARNING:tensorflow:`period` argument is deprecated. Please use `save_freq` to specify the frequency in number of batches seen.
Training: 0%
0/20 [00:00<?, ?it/s]
Epoch 0: 0%
0/237 [00:00<?, ?it/s]
Epoch 1/20
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
<ipython-input-32-0fc16141d8f6> in <module>
32 seq2seq.compile(optimizer=keras.optimizers.Adam(), loss='categorical_crossentropy')
33
---> 34 seq2seq.fit([X_train_image, X_train_text], y_train_text,
35 validation_data=([X_test_image, X_test_text], y_test_text),
36 batch_size=1024,
~\anaconda3\lib\site-packages\keras\utils\traceback_utils.py in error_handler(*args, **kwargs)
65 except Exception as e: # pylint: disable=broad-except
66 filtered_tb = _process_traceback_frames(e.__traceback__)
---> 67 raise e.with_traceback(filtered_tb) from None
68 finally:
69 del filtered_tb
~\anaconda3\lib\site-packages\keras_tqdm\tqdm_callback.py in append_logs(self, logs)
134
135 def append_logs(self, logs):
--> 136 metrics = self.params['metrics']
137 for metric, value in six.iteritems(logs):
138 if metric in metrics:
KeyError: 'metrics'
`
I'm using Python 3.7.7. and Tensorflow 2.1.0 with Functional API and Eager Execution.
I'm trying to do custom training, with an encoder extracted from a U-Net pretrained network:
I get the U-Net model without compile it.
I have loaded the weights into the model.
I have extracted the encoder and decoder from that model.
Then I want to use the encoder with this summary:
Model: "encoder"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
input_1 (InputLayer) [(None, 200, 200, 1)] 0
_________________________________________________________________
conv1_1 (Conv2D) (None, 200, 200, 64) 1664
_________________________________________________________________
conv1_2 (Conv2D) (None, 200, 200, 64) 102464
_________________________________________________________________
pool1 (MaxPooling2D) (None, 100, 100, 64) 0
_________________________________________________________________
conv2_1 (Conv2D) (None, 100, 100, 96) 55392
_________________________________________________________________
conv2_2 (Conv2D) (None, 100, 100, 96) 83040
_________________________________________________________________
pool2 (MaxPooling2D) (None, 50, 50, 96) 0
_________________________________________________________________
conv3_1 (Conv2D) (None, 50, 50, 128) 110720
_________________________________________________________________
conv3_2 (Conv2D) (None, 50, 50, 128) 147584
_________________________________________________________________
pool3 (MaxPooling2D) (None, 25, 25, 128) 0
_________________________________________________________________
conv4_1 (Conv2D) (None, 25, 25, 256) 295168
_________________________________________________________________
conv4_2 (Conv2D) (None, 25, 25, 256) 1048832
_________________________________________________________________
pool4 (MaxPooling2D) (None, 12, 12, 256) 0
_________________________________________________________________
conv5_1 (Conv2D) (None, 12, 12, 512) 1180160
_________________________________________________________________
conv5_2 (Conv2D) (None, 12, 12, 512) 2359808
=================================================================
Total params: 5,384,832
Trainable params: 5,384,832
Non-trainable params: 0
_________________________________________________________________
I use this function to do the custom training:
def train_encoder_unet_custom(model, dataset):
optimizer = tf.keras.optimizers.Adam(learning_rate=0.01)
for episode in range(num_episodes):
selected = np.random.permutation(no_of_samples)[:num_shot + num_query]
# Create our Support Set.
support_set = np.array(dataset[selected[:num_shot]])
X_train = support_set[:,0,:]
y_train = support_set[:,1,:]
loss_value, grads = grad(model, X_train, y_train)
optimizer.apply_gradients(zip(grads, model.trainable_variables))
The grad function is:
loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
def loss(model, x, y, training):
# training=training is needed only if there are layers with different
# behavior during training versus inference (e.g. Dropout).
y_ = model(x, training=training)
return loss_object(y_true=y, y_pred=y_)
def grad(model, inputs, targets):
with tf.GradientTape() as tape:
loss_value = loss(model, inputs, targets, training=False)
return loss_value, tape.gradient(loss_value, model.trainable_variables)
But when I try to run it I get the error:
InvalidArgumentError: Shapes of all inputs must match: values[0].shape = [5,12,12,512] != values[1].shape = [5,25,25,256] [Op:Pack] name: packed
In loss function, I have checked the values for y_ variable. y_ is a list of 6 elements with these shapes:
(5, 12, 12, 512)
(5, 25, 25, 256)
(5, 50, 50, 128)
(5, 100, 100, 96)
(5, 200, 200, 64)
(5, 200, 200, 1)
Any idea about what is it happening?
If you need more details, please ask me.
This is the full call stack:
<ipython-input-133-22827956a9f6> in train_encoder_unet_custom(model, dataset, feat_type, show)
22 y_valid = query_set[:,1,:]
23
---> 24 loss_value, grads = grad(model, X_train, y_train)
25
26 optimizer.apply_gradients(zip(grads, model.trainable_variables))
<ipython-input-143-58ff4de686d6> in grad(model, inputs, targets)
10 def grad(model, inputs, targets):
11 with tf.GradientTape() as tape:
---> 12 loss_value = loss(model, inputs, targets, training=False)
13 return loss_value, tape.gradient(loss_value, model.trainable_variables)
<ipython-input-143-58ff4de686d6> in loss(model, x, y, training)
6 y_ = model(x, training=training)
7
----> 8 return loss_object(y_true=y, y_pred=y_)
9
10 def grad(model, inputs, targets):
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/losses.py in __call__(self, y_true, y_pred, sample_weight)
147 with K.name_scope(self._name_scope), graph_ctx:
148 ag_call = autograph.tf_convert(self.call, ag_ctx.control_status_ctx())
--> 149 losses = ag_call(y_true, y_pred)
150 return losses_utils.compute_weighted_loss(
151 losses, sample_weight, reduction=self._get_reduction())
/usr/local/lib/python3.6/dist-packages/tensorflow/python/autograph/impl/api.py in wrapper(*args, **kwargs)
253 try:
254 with conversion_ctx:
--> 255 return converted_call(f, args, kwargs, options=options)
256 except Exception as e: # pylint:disable=broad-except
257 if hasattr(e, 'ag_error_metadata'):
/usr/local/lib/python3.6/dist-packages/tensorflow/python/autograph/impl/api.py in converted_call(f, args, kwargs, caller_fn_scope, options)
455 if conversion.is_in_whitelist_cache(f, options):
456 logging.log(2, 'Whitelisted %s: from cache', f)
--> 457 return _call_unconverted(f, args, kwargs, options, False)
458
459 if ag_ctx.control_status_ctx().status == ag_ctx.Status.DISABLED:
/usr/local/lib/python3.6/dist-packages/tensorflow/python/autograph/impl/api.py in _call_unconverted(f, args, kwargs, options, update_cache)
337
338 if kwargs is not None:
--> 339 return f(*args, **kwargs)
340 return f(*args)
341
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/losses.py in call(self, y_true, y_pred)
251 y_pred, y_true)
252 ag_fn = autograph.tf_convert(self.fn, ag_ctx.control_status_ctx())
--> 253 return ag_fn(y_true, y_pred, **self._fn_kwargs)
254
255 def get_config(self):
/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/dispatch.py in wrapper(*args, **kwargs)
199 """Call target, and fall back on dispatchers if there is a TypeError."""
200 try:
--> 201 return target(*args, **kwargs)
202 except (TypeError, ValueError):
203 # Note: convert_to_eager_tensor currently raises a ValueError, not a
/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/losses.py in sparse_categorical_crossentropy(y_true, y_pred, from_logits, axis)
1562 Sparse categorical crossentropy loss value.
1563 """
-> 1564 y_pred = ops.convert_to_tensor_v2(y_pred)
1565 y_true = math_ops.cast(y_true, y_pred.dtype)
1566 return K.sparse_categorical_crossentropy(
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py in convert_to_tensor_v2(value, dtype, dtype_hint, name)
1380 name=name,
1381 preferred_dtype=dtype_hint,
-> 1382 as_ref=False)
1383
1384
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py in convert_to_tensor(value, dtype, name, as_ref, preferred_dtype, dtype_hint, ctx, accepted_result_types)
1497
1498 if ret is None:
-> 1499 ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref)
1500
1501 if ret is NotImplemented:
/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/array_ops.py in _autopacking_conversion_function(v, dtype, name, as_ref)
1500 elif dtype != inferred_dtype:
1501 v = nest.map_structure(_cast_nested_seqs_to_dtype(dtype), v)
-> 1502 return _autopacking_helper(v, dtype, name or "packed")
1503
1504
/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/array_ops.py in _autopacking_helper(list_or_tuple, dtype, name)
1406 # checking.
1407 if all(isinstance(elem, core.Tensor) for elem in list_or_tuple):
-> 1408 return gen_array_ops.pack(list_or_tuple, name=name)
1409 must_pack = False
1410 converted_elems = []
/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/gen_array_ops.py in pack(values, axis, name)
6457 return _result
6458 except _core._NotOkStatusException as e:
-> 6459 _ops.raise_from_not_ok_status(e, name)
6460 except _core._FallbackException:
6461 pass
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py in raise_from_not_ok_status(e, name)
6841 message = e.message + (" name: " + name if name is not None else "")
6842 # pylint: disable=protected-access
-> 6843 six.raise_from(core._status_to_exception(e.code, message), None)
6844 # pylint: enable=protected-access
6845
/usr/local/lib/python3.6/dist-packages/six.py in raise_from(value, from_value)
I'm constructing an encoder-decoder using a BLSTM to model word inflection generation.
I'm not sure why I am getting the titular error message at the model.fit step. I am passing in a matrix of integer-encoded word vectors, but I was under the impression that my input would be converted to three dimensions when passed through the Embedding layer.
encoder_inputs = Input(shape=(enc_len,))
encoder_embedding = Embedding(vocab_size, 100, mask_zero=True)(encoder_inputs)
encoder_outputs = Bidirectional(LSTM(100))(encoder_embedding)
e = Dense(200)(encoder_outputs)
e = RepeatVector(35)(e)
decoder_inputs_lemm = Input(shape=(dec_len,))
decoder_inputs_infl = Input(shape=(dec_len,))
embedding_layer = Embedding(vocab_size, 100) # shared weights
decoder_embedding_lemm = embedding_layer(decoder_inputs_lemm)
decoder_embedding_infl = embedding_layer(decoder_inputs_infl)
concat = Concatenate()([decoder_embedding_lemm, decoder_embedding_infl, e])
decoder_outputs = LSTM(100, return_sequences=True)(concat)
decoder_outputs = TimeDistributed(Dense(dec_len, activation='softmax'))(decoder_outputs)
# prepare input data
enc_lemma = pad_sequences([x[0] for x in data['train']], enc_len, padding='pre')
dec_lemma = pad_sequences([x[0] for x in data['train']], dec_len, padding='post')
dec_infl_shifted = pad_sequences([x[1] for x in data['train']], enc_len, padding='post')
dec_infl_shifted = np.hstack((np.full((dec_infl_shifted.shape[0], 1), 2), dec_infl_shifted))
dec_infl_target = pad_sequences([x[1] for x in data['train']], enc_len, padding='post') # not shifted
dec_infl_target = np.hstack((dec_infl_target, np.full((dec_infl_target.shape[0], 1), 0)))
model = Model([encoder_inputs, decoder_inputs_lemm, decoder_inputs_infl], decoder_outputs)
model.compile(optimizer='adadelta', loss='categorical_crossentropy')
model.fit([enc_lemma, dec_lemma, dec_infl_shifted], dec_infl_target, epochs=30, verbose=1)
Here is the summary:
Model: "model_1"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
input_1 (InputLayer) (None, 34) 0
__________________________________________________________________________________________________
embedding_1 (Embedding) (None, 34, 100) 6300 input_1[0][0]
__________________________________________________________________________________________________
bidirectional_1 (Bidirectional) (None, 200) 160800 embedding_1[0][0]
__________________________________________________________________________________________________
input_2 (InputLayer) (None, 35) 0
__________________________________________________________________________________________________
input_3 (InputLayer) (None, 35) 0
__________________________________________________________________________________________________
dense_1 (Dense) (None, 200) 40200 bidirectional_1[0][0]
__________________________________________________________________________________________________
embedding_2 (Embedding) (None, 35, 100) 6300 input_2[0][0]
input_3[0][0]
__________________________________________________________________________________________________
repeat_vector_1 (RepeatVector) (None, 35, 200) 0 dense_1[0][0]
__________________________________________________________________________________________________
concatenate_1 (Concatenate) (None, 35, 400) 0 embedding_2[0][0]
embedding_2[1][0]
repeat_vector_1[0][0]
__________________________________________________________________________________________________
lstm_2 (LSTM) (None, 35, 100) 200400 concatenate_1[0][0]
__________________________________________________________________________________________________
time_distributed_1 (TimeDistrib (None, 35, 35) 3535 lstm_2[0][0]
==================================================================================================
Total params: 417,535
Trainable params: 417,535
Non-trainable params: 0
__________________________________________________________________________________________________
None
I'm trying to recreate the results in the Cleaning Up Dirty Scanned Documents with Deep Learning, using the ImageDataGenerator to read in the images and model.fit_generator to train my model, but when I try to fit, I get the following error and stack trace:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-34-2dedade68c7a> in <module>()
4 epochs=epochs,
5 validation_data=validation_generator,
----> 6 validation_steps=nb_validation_samples // batch_size)
/usr/local/lib/python3.6/dist-packages/keras/legacy/interfaces.py in wrapper(*args, **kwargs)
89 warnings.warn('Update your `' + object_name + '` call to the ' +
90 'Keras 2 API: ' + signature, stacklevel=2)
---> 91 return func(*args, **kwargs)
92 wrapper._original_function = func
93 return wrapper
/usr/local/lib/python3.6/dist-packages/keras/engine/training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
1656 use_multiprocessing=use_multiprocessing,
1657 shuffle=shuffle,
-> 1658 initial_epoch=initial_epoch)
1659
1660 #interfaces.legacy_generator_methods_support
/usr/local/lib/python3.6/dist-packages/keras/engine/training_generator.py in fit_generator(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
213 outs = model.train_on_batch(x, y,
214 sample_weight=sample_weight,
--> 215 class_weight=class_weight)
216
217 outs = to_list(outs)
/usr/local/lib/python3.6/dist-packages/keras/engine/training.py in train_on_batch(self, x, y, sample_weight, class_weight)
1441 x, y,
1442 sample_weight=sample_weight,
-> 1443 class_weight=class_weight)
1444 if self._uses_dynamic_learning_phase():
1445 ins = x + y + sample_weights + [1.]
/usr/local/lib/python3.6/dist-packages/keras/engine/training.py in _standardize_user_data(self, x, y, sample_weight, class_weight, check_array_lengths, batch_size)
793 feed_output_shapes,
794 check_batch_axis=False, # Don't enforce the batch size.
--> 795 exception_prefix='target')
796
797 # Generate sample-wise weight values given the `sample_weight` and
/usr/local/lib/python3.6/dist-packages/keras/engine/training_utils.py in standardize_input_data(data, names, shapes, check_batch_axis, exception_prefix)
139 ': expected ' + names[i] + ' to have shape ' +
140 str(shape) + ' but got array with shape ' +
--> 141 str(data_shape))
142 return data
143
ValueError: Error when checking target: expected conv2d_10 to have shape (1, 1, 1) but got array with shape (258, 540, 3)
Here is my script:
from keras.models import Sequential
from keras.preprocessing.image import ImageDataGenerator
from keras.layers import Conv2D, Activation, BatchNormalization, LeakyReLU, MaxPooling2D, UpSampling2D
import numpy as np
from keras import backend as K
img_width, img_height = 258,540
train_data_dir = 'drive/My Drive/train'
validation_data_dir = 'drive/My Drive/train_cleaned'
nb_train_samples = 144
nb_validation_samples = 144
epochs = 200
batch_size = 20
if K.image_data_format() == 'channels_first':
input_shape = (3, img_width, img_height)
else:
input_shape = (img_width, img_height, 3)
model = Sequential([
Conv2D(input_shape=input_shape, filters=64, kernel_size=(258, 540), padding='same'),
LeakyReLU(),
BatchNormalization(),
Conv2D(filters=64, kernel_size=(258, 540), padding='same'),
LeakyReLU(),
MaxPooling2D((2,2), padding='same'),
Conv2D(filters=64, kernel_size=(129, 270), padding='same'),
LeakyReLU(),
BatchNormalization(),
Conv2D(filters=64, kernel_size=(129, 270), padding='same'),
LeakyReLU(),
UpSampling2D((2, 2)),
Conv2D(filters=1, kernel_size=(258,540), activation='sigmoid')
])
model.compile(optimizer='adam', loss='mse', metrics=['accuracy'])
train_datagen = ImageDataGenerator(
rescale=1. / 255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
test_datagen = ImageDataGenerator(rescale=1. / 255)
train_generator = train_datagen.flow_from_directory(
train_data_dir,
target_size=(img_width, img_height),
class_mode='input',
batch_size=batch_size)
validation_generator = test_datagen.flow_from_directory(
validation_data_dir,
target_size=(img_width, img_height),
class_mode='input',
batch_size=batch_size)
model.fit_generator(
train_generator,
steps_per_epoch=nb_train_samples // batch_size,
epochs=epochs,
validation_data=validation_generator,
validation_steps=nb_validation_samples // batch_size)
The output of model.summary() can be found below:
Model: "sequential_2"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_6 (Conv2D) (None, 258, 540, 64) 26749504
_________________________________________________________________
leaky_re_lu_5 (LeakyReLU) (None, 258, 540, 64) 0
_________________________________________________________________
batch_normalization_3 (Batch (None, 258, 540, 64) 256
_________________________________________________________________
conv2d_7 (Conv2D) (None, 258, 540, 64) 570654784
_________________________________________________________________
leaky_re_lu_6 (LeakyReLU) (None, 258, 540, 64) 0
_________________________________________________________________
max_pooling2d_2 (MaxPooling2 (None, 129, 270, 64) 0
_________________________________________________________________
conv2d_8 (Conv2D) (None, 129, 270, 64) 142663744
_________________________________________________________________
leaky_re_lu_7 (LeakyReLU) (None, 129, 270, 64) 0
_________________________________________________________________
batch_normalization_4 (Batch (None, 129, 270, 64) 256
_________________________________________________________________
conv2d_9 (Conv2D) (None, 129, 270, 64) 142663744
_________________________________________________________________
leaky_re_lu_8 (LeakyReLU) (None, 129, 270, 64) 0
_________________________________________________________________
up_sampling2d_2 (UpSampling2 (None, 258, 540, 64) 0
_________________________________________________________________
conv2d_10 (Conv2D) (None, 1, 1, 1) 8916481
=================================================================
Total params: 891,648,769
Trainable params: 891,648,513
Non-trainable params: 256
_________________________________________________________________
None
I'm not very familiar with Keras and it seems like the error is with my last layer's input shape, but I don't know how to change the input size that's expected. I tried passing in the parameter input_shape=input_shape to the last layer and that didn't change anything.
I'd appreciate any other critique on the code that I've written, even if it's not directly related to answering my question. Thanks!
EDIT: Here is an image of the network I'm trying to recreate:
Change all these:
kernel_size=(258, 540)
to this:
kernel_size=(3, 3)
Keras is confused because your convolutional filters are the same size than your images. So after one iteration your input is effectively (1, 1, 1). That is because the filters slide onto the input image, multiplying every corresponding pixel, and then summing up. If the pixels and filters are the same size, all pixels and filter units will be multiplied, and summed to one number.
Even after applying the suggestions in answer and comments, it looks like the dimension mismatch issue persists. This is exact code and data file to replicate as well: https://drive.google.com/drive/folders/1q67s0VhB-O7J8OtIhU2jmj7Kc4LxL3sf?usp=sharing
How can this be corrected!? Latest code, model summary, functions used and error I get is below
type_ae=='dcor'
#Wrappers for keras
def custom_loss1(y_true,y_pred):
dcor = -1*distance_correlation(y_true,encoded_layer)
return dcor
def custom_loss2(y_true,y_pred):
recon_loss = losses.categorical_crossentropy(y_true, y_pred)
return recon_loss
input_layer = Input(shape=(64,64,1))
encoded_layer = Conv2D(filters = 128, kernel_size = (5,5),padding = 'same',activation ='relu',
input_shape = (64,64,1))(input_layer)
encoded_layer = MaxPool2D(pool_size=(2,2))(encoded_layer)
encoded_layer = Dropout(0.25)(encoded_layer)
encoded_layer = (Conv2D(filters = 64, kernel_size = (3,3),padding = 'same',activation ='relu'))(encoded_layer)
encoded_layer = (MaxPool2D(pool_size=(2,2)))(encoded_layer)
encoded_layer = (Dropout(0.25))(encoded_layer)
encoded_layer = (Conv2D(filters = 64, kernel_size = (3,3),padding = 'same',activation ='relu'))(encoded_layer)
encoded_layer = (MaxPool2D(pool_size=(2,2)))(encoded_layer)
encoded_layer = (Dropout(0.25))(encoded_layer)
encoded_layer = Conv2D(filters = 1, kernel_size = (3,3),padding = 'same',activation ='relu',
input_shape = (64,64,1),strides=1)(encoded_layer)
encoded_layer = ZeroPadding2D(padding=(28, 28), data_format=None)(encoded_layer)
decoded_imag = Conv2D(8, (2, 2), activation='relu', padding='same')(encoded_layer)
decoded_imag = UpSampling2D((2, 2))(decoded_imag)
decoded_imag = Conv2D(8, (3, 3), activation='relu', padding='same')(decoded_imag)
decoded_imag = UpSampling2D((2, 2))(decoded_imag)
decoded_imag = Conv2D(16, (3, 3), activation='relu', padding='same')(decoded_imag)
decoded_imag = UpSampling2D((2, 2))(decoded_imag)
decoded_imag = Conv2D(1, (3, 3), activation='sigmoid', padding='same')(decoded_imag)
flat_layer = Flatten()(decoded_imag)
dense_layer = Dense(256,activation = "relu")(flat_layer)
dense_layer = Dense(64,activation = "relu")(dense_layer)
dense_layer = Dense(32,activation = "relu")(dense_layer)
output_layer = Dense(9, activation = "softmax")(dense_layer)
autoencoder = Model(input_layer, [encoded_layer,output_layer])
autoencoder.summary()
autoencoder.compile(optimizer='adadelta', loss=[custom_loss1,custom_loss2])
autoencoder.fit(x_train,[x_train, y_train],batch_size=32,epochs=3,shuffle=True,
validation_data=(x_val, [x_val,y_val]))
The data is of dimensions:
x_train.shape: (4000, 64, 64, 1)
x_val.shape: (1000, 64, 64, 1)
y_train.shape: (4000, 9)
y_val.shape: (1000, 9)
losses look like:
def custom_loss1(y_true,y_pred):
dcor = -1*distance_correlation(y_true,encoded_layer)
return dcor
def custom_loss2(y_true,y_pred):
recon_loss = losses.categorical_crossentropy(y_true, y_pred)
return recon_loss
The correlation function is based on tensors as follows:
def distance_correlation(y_true,y_pred):
pred_r = tf.reduce_sum(y_pred*y_pred,1)
pred_r = tf.reshape(pred_r,[-1,1])
pred_d = pred_r - 2*tf.matmul(y_pred,tf.transpose(y_pred))+tf.transpose(pred_r)
true_r = tf.reduce_sum(y_true*y_true,1)
true_r = tf.reshape(true_r,[-1,1])
true_d = true_r - 2*tf.matmul(y_true,tf.transpose(y_true))+tf.transpose(true_r)
concord = 1-tf.matmul(y_true,tf.transpose(y_true))
#print(pred_d)
#print(tf.reshape(tf.reduce_mean(pred_d,1),[-1,1]))
#print(tf.reshape(tf.reduce_mean(pred_d,0),[1,-1]))
#print(tf.reduce_mean(pred_d))
tf.check_numerics(pred_d,'pred_d has NaN')
tf.check_numerics(true_d,'true_d has NaN')
A = pred_d - tf.reshape(tf.reduce_mean(pred_d,1),[-1,1]) - tf.reshape(tf.reduce_mean(pred_d,0),[1,-1]) + tf.reduce_mean(pred_d)
B = true_d - tf.reshape(tf.reduce_mean(true_d,1),[-1,1]) - tf.reshape(tf.reduce_mean(true_d,0),[1,-1]) + tf.reduce_mean(true_d)
#dcor = -tf.reduce_sum(concord*pred_d)/tf.reduce_sum((1-concord)*pred_d)
dcor = -tf.log(tf.reduce_mean(A*B))+tf.log(tf.sqrt(tf.reduce_mean(A*A)*tf.reduce_mean(B*B)))#-tf.reduce_sum(concord*pred_d)/tf.reduce_sum((1-concord)*pred_d)
#print(dcor.shape)
#tf.Print(dcor,[dcor])
#dcor = tf.tile([dcor],batch_size)
return (dcor)
model summary looks like:
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
input_5 (InputLayer) (None, 64, 64, 1) 0
_________________________________________________________________
conv2d_30 (Conv2D) (None, 64, 64, 128) 3328
_________________________________________________________________
max_pooling2d_13 (MaxPooling (None, 32, 32, 128) 0
_________________________________________________________________
dropout_13 (Dropout) (None, 32, 32, 128) 0
_________________________________________________________________
conv2d_31 (Conv2D) (None, 32, 32, 64) 73792
_________________________________________________________________
max_pooling2d_14 (MaxPooling (None, 16, 16, 64) 0
_________________________________________________________________
dropout_14 (Dropout) (None, 16, 16, 64) 0
_________________________________________________________________
conv2d_32 (Conv2D) (None, 16, 16, 64) 36928
_________________________________________________________________
max_pooling2d_15 (MaxPooling (None, 8, 8, 64) 0
_________________________________________________________________
dropout_15 (Dropout) (None, 8, 8, 64) 0
_________________________________________________________________
conv2d_33 (Conv2D) (None, 8, 8, 1) 577
_________________________________________________________________
zero_padding2d_5 (ZeroPaddin (None, 64, 64, 1) 0
_________________________________________________________________
conv2d_34 (Conv2D) (None, 64, 64, 8) 40
_________________________________________________________________
up_sampling2d_10 (UpSampling (None, 128, 128, 8) 0
_________________________________________________________________
conv2d_35 (Conv2D) (None, 128, 128, 8) 584
_________________________________________________________________
up_sampling2d_11 (UpSampling (None, 256, 256, 8) 0
_________________________________________________________________
conv2d_36 (Conv2D) (None, 256, 256, 16) 1168
_________________________________________________________________
up_sampling2d_12 (UpSampling (None, 512, 512, 16) 0
_________________________________________________________________
conv2d_37 (Conv2D) (None, 512, 512, 1) 145
_________________________________________________________________
flatten_4 (Flatten) (None, 262144) 0
_________________________________________________________________
dense_13 (Dense) (None, 256) 67109120
_________________________________________________________________
dense_14 (Dense) (None, 64) 16448
_________________________________________________________________
dense_15 (Dense) (None, 32) 2080
_________________________________________________________________
dense_16 (Dense) (None, 9) 297
=================================================================
Total params: 67,244,507
Trainable params: 67,244,507
Non-trainable params: 0
_________________________________________________________________
This is the error:
InvalidArgumentError Traceback (most recent call last)
~/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py in _create_c_op(graph, node_def, inputs, control_inputs)
1658 try:
-> 1659 c_op = c_api.TF_FinishOperation(op_desc)
1660 except errors.InvalidArgumentError as e:
InvalidArgumentError: Dimensions must be equal, but are 1 and 64 for 'loss_1/zero_padding2d_5_loss/MatMul' (op: 'BatchMatMul') with input shapes: [?,64,64,1], [1,64,64,?].
During handling of the above exception, another exception occurred:
ValueError Traceback (most recent call last)
<ipython-input-11-0e924885fc6b> in <module>
40 autoencoder = Model(input_layer, [encoded_layer,output_layer])
41 autoencoder.summary()
---> 42 autoencoder.compile(optimizer='adadelta', loss=[custom_loss1,custom_loss2])
43 autoencoder.fit(x_train,[x_train, y_train],batch_size=32,epochs=3,shuffle=True,
44 validation_data=(x_val, [x_val,y_val]))
~/anaconda3/lib/python3.6/site-packages/keras/engine/training.py in compile(self, optimizer, loss, metrics, loss_weights, sample_weight_mode, weighted_metrics, target_tensors, **kwargs)
340 with K.name_scope(self.output_names[i] + '_loss'):
341 output_loss = weighted_loss(y_true, y_pred,
--> 342 sample_weight, mask)
343 if len(self.outputs) > 1:
344 self.metrics_tensors.append(output_loss)
~/anaconda3/lib/python3.6/site-packages/keras/engine/training_utils.py in weighted(y_true, y_pred, weights, mask)
402 """
403 # score_array has ndim >= 2
--> 404 score_array = fn(y_true, y_pred)
405 if mask is not None:
406 # Cast the mask to floatX to avoid float64 upcasting in Theano
<ipython-input-11-0e924885fc6b> in custom_loss1(y_true, y_pred)
2 #Wrappers for keras
3 def custom_loss1(y_true,y_pred):
----> 4 dcor = -1*distance_correlation(y_true,encoded_layer)
5 return dcor
6
<ipython-input-6-f282528532cc> in distance_correlation(y_true, y_pred)
2 pred_r = tf.reduce_sum(y_pred*y_pred,1)
3 pred_r = tf.reshape(pred_r,[-1,1])
----> 4 pred_d = pred_r - 2*tf.matmul(y_pred,tf.transpose(y_pred))+tf.transpose(pred_r)
5 true_r = tf.reduce_sum(y_true*y_true,1)
6 true_r = tf.reshape(true_r,[-1,1])
~/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py in matmul(a, b, transpose_a, transpose_b, adjoint_a, adjoint_b, a_is_sparse, b_is_sparse, name)
2415 adjoint_b = True
2416 return gen_math_ops.batch_mat_mul(
-> 2417 a, b, adj_x=adjoint_a, adj_y=adjoint_b, name=name)
2418
2419 # Neither matmul nor sparse_matmul support adjoint, so we conjugate
~/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gen_math_ops.py in batch_mat_mul(x, y, adj_x, adj_y, name)
1421 adj_y = _execute.make_bool(adj_y, "adj_y")
1422 _, _, _op = _op_def_lib._apply_op_helper(
-> 1423 "BatchMatMul", x=x, y=y, adj_x=adj_x, adj_y=adj_y, name=name)
1424 _result = _op.outputs[:]
1425 _inputs_flat = _op.inputs
~/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py in _apply_op_helper(self, op_type_name, name, **keywords)
786 op = g.create_op(op_type_name, inputs, output_types, name=scope,
787 input_types=input_types, attrs=attr_protos,
--> 788 op_def=op_def)
789 return output_structure, op_def.is_stateful, op
790
~/anaconda3/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py in new_func(*args, **kwargs)
505 'in a future version' if date is None else ('after %s' % date),
506 instructions)
--> 507 return func(*args, **kwargs)
508
509 doc = _add_deprecated_arg_notice_to_docstring(
~/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py in create_op(***failed resolving arguments***)
3298 input_types=input_types,
3299 original_op=self._default_original_op,
-> 3300 op_def=op_def)
3301 self._create_op_helper(ret, compute_device=compute_device)
3302 return ret
~/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py in __init__(self, node_def, g, inputs, output_types, control_inputs, input_types, original_op, op_def)
1821 op_def, inputs, node_def.attr)
1822 self._c_op = _create_c_op(self._graph, node_def, grouped_inputs,
-> 1823 control_input_ops)
1824
1825 # Initialize self._outputs.
~/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py in _create_c_op(graph, node_def, inputs, control_inputs)
1660 except errors.InvalidArgumentError as e:
1661 # Convert to ValueError for backwards compatibility.
-> 1662 raise ValueError(str(e))
1663
1664 return c_op
ValueError: Dimensions must be equal, but are 1 and 64 for 'loss_1/zero_padding2d_5_loss/MatMul' (op: 'BatchMatMul') with input shapes: [?,64,64,1], [1,64,64,?].
You are having two loss functions and so you have to pass two y (ground truths) for evaluating the loss with respect to the predictions.
Your first prediction is the output of layer encoded_layer which has a size of (None, 8, 8, 128) as observed from the model.summary for conv2d_59 (Conv2D)
But what you are passing in the fit for y is [x_train, y_train]. The loss_1 is expecting input of size (None, 8, 8, 128) but you are passing x_train which has a different size.
If you want the loss_1 to find the correlation of input image with the encoded image then stack the convolutions such that the output of the convolutions will result in the shape which is the same as your x_train image shape. Use model.summary to see the output shape of convolutions.
No use the padding, strides and kernel size of the convolution layer to get the desired output size of convolutions. use formula W2=(W1−F+2P)/S+1 and H2=(H1−F+2P)/S+1 to find the output width and height of convolutions. Check this reference
There are two major issues with your approach.
Your loss function is checking the correlation between the encoded image and the actual image. The correct way to do it is to decode the image back from the encoded image and then check the correlation between the decoded image and the actual image (in lines of Autoencoder)
Your loss 1 is using numpy arrays. For a loss function to be part of a computation graph it should use tensor operations, not numy operations.
Below is the working code. However, for loss 1 I am using l2 norm of the two images. If you want to use correlation then you have to somehow convert it into tensor operations (which is a different issue from this question)
def image_loss(y_true,y_pred):
return tf.norm(y_true - y_pred)
def label_loss(y_true,y_pred):
return categorical_crossentropy(y_true, y_pred)
input_img = Input(shape=(64, 64, 1))
enocded_imag = Conv2D(16, (3, 3), activation='relu', padding='same')(input_img)
enocded_imag = MaxPooling2D((2, 2), padding='same')(enocded_imag)
enocded_imag = Conv2D(8, (3, 3), activation='relu', padding='same')(enocded_imag)
enocded_imag = MaxPooling2D((2, 2), padding='same')(enocded_imag)
enocded_imag = Conv2D(8, (3, 3), activation='relu', padding='same')(enocded_imag)
enocded_imag = MaxPooling2D((2, 2), padding='same')(enocded_imag)
decoded_imag = Conv2D(8, (2, 2), activation='relu', padding='same')(enocded_imag)
decoded_imag = UpSampling2D((2, 2))(decoded_imag)
decoded_imag = Conv2D(8, (3, 3), activation='relu', padding='same')(decoded_imag)
decoded_imag = UpSampling2D((2, 2))(decoded_imag)
decoded_imag = Conv2D(16, (3, 3), activation='relu', padding='same')(decoded_imag)
decoded_imag = UpSampling2D((2, 2))(decoded_imag)
decoded_imag = Conv2D(1, (3, 3), activation='sigmoid', padding='same')(decoded_imag)
flat_layer = Flatten()(enocded_imag)
dense_layer = Dense(32,activation = "relu")(flat_layer)
output_layer = Dense(9, activation = "softmax")(dense_layer)
model = Model(input_img, [decoded_imag, output_layer])
model.compile(optimizer='adadelta', loss=[image_loss, label_loss])
images = np.random.randn(10,64,64,1)
model.fit(images, [images, np.random.randn(10,9)])
The loss function distance_correlation you have coded assumes that each row in y_true and y_pred represent an image. When you use Dense layers it will work because Dense layer outputs a batch of (row) vectors, where each vector represents an individual image. However, 2D convolutions operate on a batch of 2d tensors with multiple channels ( you have only 1 channel). So to use the distance_correlation loss function you have to reshape your tensor such that each row corresponds to an image. Add below two lines to reshape your tensors.
def distance_correlation(y_true,y_pred):
y_true = tf.reshape(tf.squeeze(y_true), [-1,64*64])
y_pred = tf.reshape(tf.squeeze(y_pred), [-1,64*64])
.... REST OF THE CODE ....
The intention is to use the original image in custom_loss1 and the scalar label values in custom_loss2. I think the working code by #mujjiga in his answer is almost correct. I suggest one slight modification.
In model.compile() pass the input tensor in the loss which needs it. Keep the other one same. model.fit() just passes the labels.
model.compile(optimizer='adadelta', loss=[custom_loss1(input_layer), custom_loss2])
mode.fit(x_train, y_train)
Inside the custom loss functions:
def custom_loss1(input):
def loss1(y_true, y_pred):
return tf.norm(input - y_pred) # use your custom loss 1
return loss1
def custom_loss2(y_true, y_pred):
return categorical_crossentropy(y_true, y_pred) # use your custom loss 2
Try this with simple in-built Keras loss functions first. If that works well, look into your custom loss functions.