I have a siamese network and I want to perform a grid seach on it using GridSearchCV.
So I create a model using the following function:
def createMod(learn_rate=0.01, optimizer='Adam'):
#K.clear_session()
# network definition
base_network = create_base_network(input_shape)
input_a = Input(shape=input_shape)
input_b = Input(shape=input_shape)
# because we re-use the same instance `base_network`,
# the weights of the network will be shared across the two branches
processed_a = base_network(input_a)
processed_b = base_network(input_b)
distance = Lambda(euclidean_distance,
output_shape=eucl_dist_output_shape)([processed_a, processed_b])
prediction = Dense(1,activation='sigmoid')(distance)
model = Model([input_a, input_b], prediction)
if(optimizer=='SGD'):
opt = SGD(lr=learn_rate)
elif (optimizer=='RMSprop'):
opt = RMSprop(lr=learn_rate)
else:
opt = Adam(lr=learn_rate)
model.compile(loss='binary_crossentropy', optimizer=opt, metrics=[accuracy])
return model
And then I create the KerasClassifier and GridSearch as follows:
model = KerasClassifier(build_fn=createMod, verbose=0)
param_grid = dict(epochs=epochs, batch_size=batch_size, learn_rate=learn_rate,optimizer=optimizer)
grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=5)
X = [tr_pairs[:, 0], tr_pairs[:, 1]]
grid_result = grid.fit(X, tr_y)
However this throws the following value error:
ValueError: Found input variables with inconsistent numbers of samples: [2, 1054]
The shape of tr_pairs[:, 0] and tr_pairs[:, 1] is (1054, 6) and tr_y is (1054,)
The code for the base networks is:
def create_base_network(input_shape):
K.clear_session()
encoder = build_encoder(latent_dim, n_in)
decoder = build_decoder(latent_dim, n_in)
item = Input(shape=(n_in, ))
encoded_repr = encoder(item)
reconstructed_item = decoder(encoded_repr)
autoencoder = Model(item, reconstructed_item)
return autoencoder
The code for the encoder and decoder are:
def build_encoder(latent_dim, input_dim):
input_layer = Input(shape=(input_dim, ))
h = Dense(32, activation='relu', activity_regularizer=regularizers.l1(10e-5))(input_layer)
h = Dropout(0.1)(h)
h = Dense(64, activation='relu')(h)
h = Dropout(0.1)(h)
# h = Dense(128, activation='relu')(h)
# h = Dropout(0.1)(h)
latent_repr = Dense(latent_dim, activation='relu')(h)
return Model(input_layer, latent_repr)
def build_decoder(latent_dim, input_dim):
model = Sequential()
# model.add(Dense(128, input_dim=latent_dim, activation='relu'))
# model.add(Dropout(0.1))
model.add(Dense(64, activation='relu'))
model.add(Dropout(0.1))
model.add(Dense(32, activation='relu'))
model.add(Dropout(0.1))
model.add(Dense(input_dim, activation='relu'))
z = Input(shape=(latent_dim,))
item = model(z)
return Model(z, item)
The code works when I do the normal keras model's .fit function but doesn't work here.. is there a problem somewhere in my code or is it just not possible to feed multiple inputs in Grid Search and if that is the case is there a way I can still perform the grid search?
this is workaround to pass multiple input. I create a dummy model that receives a SINGLE input in the format (n_sample, 2, 6) and then split it into two parts using Lambda layer. you can modify this according to your siamese structure.
def createMod(optimizer='Adam'):
combi_input = Input((2,6)) # (n_sample, 2, 6)
input_a = Lambda(lambda x: x[:,0])(combi_input) # (n_sample, 6)
input_b = Lambda(lambda x: x[:,1])(combi_input) # (n_sample, 6)
c = Concatenate()([input_a,input_b])
x = Dense(32)(c)
prediction = Dense(1,activation='sigmoid')(x)
model = Model(combi_input, prediction)
model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics='accuracy')
return model
tr_pairs = np.random.uniform(0,1, (1054, 2, 6))
tr_y = np.random.randint(0,2, 1054)
model = tf.keras.wrappers.scikit_learn.KerasClassifier(build_fn=createMod, verbose=0)
batch_size = [10, 20]
epochs = [10, 5]
optimizer = ['adam','SGD']
param_grid = dict(batch_size=batch_size, epochs=epochs)
grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)
grid_result = grid.fit(tr_pairs, tr_y)
Related
it gives another error.
The first argument to Layer.call must always be passed.
I cannot solve the problem. input_dim cannot be set as a constant. PCA and SelectKBest will cut down on the amount of input.
And if you can help with the output of the results from the pipeline, I will be very grateful
attach a link to the data: https://1drv.ms/u/s!AlHgQsqCKEIPiIxzdyWE0BfBHNocTQ?e=cxuSuo
def modelReg(inpt, opt = 'adam', kInitializer = 'glorot_uniform', dropout = 0.05):
model = Sequential()
model.add(Dense(1024, activation='relu', input_dim = inpt, kernel_initializer=kInitializer))
model.add(Dense(1024, activation='relu', kernel_initializer=kInitializer))
model.add(Dense(512, activation='relu', kernel_initializer=kInitializer))
model.add(layers.Dropout(dropout))
model.add(Dense(1, activation='sigmoid', kernel_initializer=kInitializer))
model.compile(loss='mse',optimizer=opt, metrics=["mse", "mae"])
return model
features = []
features.append(('pca', PCA(n_components=10)))
features.append(('select_best', SelectKBest(k=10)))
feature_union = FeatureUnion(features)
regressor = KerasRegressor(build_fn = modelReg(inpt), epochs = 3, batch_size = 500, verbose = 1)
estimators = []
estimators.append(('standardize', StandardScaler()))
estimators.append(('feature_union', feature_union))
estimators.append(('regressor' regressor))
model = Pipeline(estimators)
model.fit(allData.drop(['VancouverH'], axis = 1), allData['VancouverH'])
in KerasRegressor with a function to pass arguments to the model function, they are written to the KerasRegressor arguments.
kearsEstimator = ('kR', KerasRegressor(createModel, inpt = trainDataX.shape[1],
epochs = 5, batch_size = 180, verbose = 1))
like this, not like this:
kearsEstimator = ('kR', KerasRegressor(createModel(inpt),
epochs = 5, batch_size = 180, verbose = 1))
well, and transferred the pipeline to the Grid. And the names of the parameters for the grid are written with the prefix.
estimators = []
estimators.append((kearsEstimator))
param_grid = {
'kR__optimizer':['adam'] #'RMSprop', 'Adam', 'Adamax', 'sgd'
}
grid = GridSearchCV(Pipeline(estimators), param_grid, cv = 5)
grid.fit(trainDataX, trainDataY)
I have a pandas dataframe of features and samples, and a single series with binary category (0 or 1) values. With that I'm trying to train a neural network, but I am getting the error:
TensorFlow incompatible shapes binary classification
Here is a summary of the code:
X_train, X_test, y_train, y_test = train_test_split(df_x, series_y, random_state=1, test_size=0.25)
best_weight_path = 'best_weights.hdf5'
x = df_x.to_numpy()
y = series_y.to_numpy()
numpy_x_train = X_train.to_numpy()
numpy_y_train = y_train.to_numpy()
numpy_x_test = X_test.to_numpy()
numpy_y_test = y_test.to_numpy()
model = Sequential()
model.add(Dense(20, input_dim=x.shape[1], activation='relu'))
model.add(Dense(10, activation='relu'))
model.add(Dense(2, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam')
monitor = EarlyStopping(monitor='val_loss', min_delta=1e-3, patience=5, verbose=1, mode='auto')
checkpointer = ModelCheckpoint(filepath=best_weight_path, verbose=0, save_best_only=True)
model.fit(x, y, validation_data=(numpy_x_test, numpy_y_test), callbacks=[monitor, checkpointer], verbose=0, epochs=1000)
ValueError: Shapes (None, 1) and (None, 2) are incompatible
Shouldn't the last dense layer have 2 units as there are two possible outcomes, so where is the shape (None, 1) coming from?
The problem is related to the correct choice of an appropriate loss function according to the format of your labels. you have 2 possibilities when using softmax in classification task:
1 possibility: if you have 1D integer encoded target, you can use sparse_categorical_crossentropy as loss function (this seems to be your case)
n_class = 2
n_features = 100
n_sample = 1000
X = np.random.randint(0,10, (n_sample,n_features))
y = np.random.randint(0,n_class, n_sample)
inp = Input((n_features,))
x = Dense(128, activation='relu')(inp)
out = Dense(n_class, activation='softmax')(x)
model = Model(inp, out)
model.compile(loss='sparse_categorical_crossentropy',optimizer='adam',metrics=['accuracy'])
history = model.fit(X, y, epochs=3)
2 possibility: if you have one-hot encoded your target in order to have 2D shape (n_samples, n_class), you can use categorical_crossentropy
n_class = 2
n_features = 100
n_sample = 1000
X = np.random.randint(0,10, (n_sample,n_features))
y = pd.get_dummies(np.random.randint(0,n_class, n_sample)).values
inp = Input((n_features,))
x = Dense(128, activation='relu')(inp)
out = Dense(n_class, activation='softmax')(x)
model = Model(inp, out)
model.compile(loss='categorical_crossentropy',optimizer='adam',metrics=['accuracy'])
history = model.fit(X, y, epochs=3)
I am experimenting with the use of multi-input to a double CNN.
However, I am getting the Graph disconnected: cannot obtain value for tensor Tensor("embedding_1_input:0", shape=(?, 40), dtype=float32) at layer "embedding_1_input". The following previous layers were accessed without issue: []
I didn't find a fix for it, knowing that there are layers with shape = (?, 50) why is there such dimension "?" ????
from keras.models import Model
from keras.layers import Input, Dot
from keras.layers.core import Reshape
from keras.backend import int_shape
max_words=50
thedata = [' '.join(list(x)) for x in self.input_data]
emb_size = 15
tokenizer = Tokenizer(num_words=max_words)
tokenizer.fit_on_texts(thedata)
self.dictionary = tokenizer.word_index
vocab_size = len(tokenizer.word_index) + 1
vocab_size = 5000
allWordIndices = []
for text in thedata:
wordIndices = self.convert_text_to_index_array(text)
allWordIndices.append(wordIndices)
allWordIndices = np.asarray(allWordIndices)
train_x = tokenizer.sequences_to_matrix(allWordIndices, mode='binary')
train_y = list(map(lambda x: self.c[x], self.output_data))
X_train, X_test, Y_train, Y_test = train_test_split(train_x, train_y, test_size = 0.1, shuffle=True)
vocab_size1 = 50000
input_size = 40
input_shape = (input_size, 1)
# first model
L_branch = Sequential()
L_branch.add(Embedding(vocab_size1, emb_size, input_length=max_words, trainable=True))
L_branch.add(Conv1D(50, activation='relu', kernel_size=10, input_shape=input_shape))
L_branch.add(MaxPooling1D(emb_size))
L_branch.add(Flatten())
L_branch.add(Dense(emb_size, activation='sigmoid'))
# second model
R_branch = Sequential()
R_branch.add(Embedding(vocab_size, output_dim=emb_size, input_length=max_words, trainable=True))
R_branch.add(Dense(emb_size, activation='sigmoid'))
input_in1 = Input(shape=(input_size, 1, 1))
input_in2 = Input(shape=(input_size, 1, 1))
merged = Dot(axes=(1, 2))([L_branch.outputs[0], R_branch.outputs[0]])
print(merged.shape)
#
merged = Reshape((-1, int_shape(merged)[1],))(merged)
latent = Conv1D(50, activation='relu', kernel_size=self.nb_classes, input_shape=(1, input_size, 1))(merged)
latent = MaxPooling1D(self.nb_classes)(latent)
out = Dense(self.nb_classes, activation='sigmoid')(latent)
final_model = Model([input_in2, input_in1], out)
final_model.compile(
loss='mse',
optimizer='adam',
metrics=['accuracy'])
final_model.summary()
final_model.fit(
[X_train, train_x],
Y_train,
batch_size=200,
epochs=5,
verbose=1
# validation_split=0.1
)
I tried adding flatten layer after the second embedding but I got an error. Even with a dropout.
I also tried eliminating the maxpooling from the L_branch model.
Tried not working with the reshape and just expand the input dims since I was getting an error of the second Conv1D saying the layer expects dims = 3 but it was getting dims = 2.
latent = Conv1D(50, activation='relu', kernel_size=nb_classes, input_shape=(1, input_size, 1))(merged)
File "/root/anaconda3/envs/oea/lib/python3.7/site-packages/keras/engine/base_layer.py", line 414, in __call__
self.assert_input_compatibility(inputs)
File "/root/anaconda3/envs/oea/lib/python3.7/site-packages/keras/engine/base_layer.py", line 311, in assert_input_compatibility
str(K.ndim(x)))
ValueError: Input 0 is incompatible with layer conv1d_2: expected ndim=3, found ndim=2`
I also didn't get which input is used in the first model and which input is used in the second?
you have to define your new model in this way: final_model = Model([L_branch.input, R_branch.input], out)
I provide an example of network with a structure similar to yours
vocab_size1 = 5000
vocab_size2 = 50000
input_size1 = 40
input_size2 = 40
max_words=50
emb_size = 15
nb_classes = 10
# first model
L_branch = Sequential()
L_branch.add(Embedding(vocab_size1, emb_size, input_length=input_size1, trainable=True))
L_branch.add(Conv1D(50, activation='relu', kernel_size=10))
L_branch.add(MaxPooling1D())
L_branch.add(Flatten())
L_branch.add(Dense(emb_size, activation='relu'))
# second model
R_branch = Sequential()
R_branch.add(Embedding(vocab_size2, emb_size, input_length=input_size2, trainable=True))
R_branch.add(Flatten())
R_branch.add(Dense(emb_size, activation='relu'))
merged = Concatenate()([L_branch.output, R_branch.output])
latent = Dense(50, activation='relu')(merged)
out = Dense(nb_classes, activation='softmax')(latent)
final_model = Model([L_branch.input, R_branch.input], out)
final_model.compile(
loss='sparse_categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
final_model.summary()
X1 = np.random.randint(0,vocab_size1, (100,input_size1))
X2 = np.random.randint(0,vocab_size2, (100,input_size2))
y = np.random.randint(0,nb_classes, 100)
final_model.fit([X1,X2], y, epochs=10)
I'm pretty new to Keras and LSTMs. I've been trying to train my model of sequences to predict the future price of a stock with the code below but the error above kept popping up.
I have tried changing the dtypes of both x_data, y_data with .astype(np.float16). However, all times I am returned with the TypeError stating that I have a float32 type.
If it helps, here are the shapes of my data:
xtrain.shape : (32, 24, 67), ytrain.shape : (32, 24, 1), xtest.shape
: (38, 67), ytest.shape : (38, 1)
Does anyone have any idea on what might be wrong? I've been stuck at this for awhile. It would be great if someone could give me a hint.
y_data = y_data.to_numpy().astype(np.float32)
x_data = main_df.to_numpy().astype(np.float32)
num_x_signals = x_data.shape[1]
num_y_signals = y_data.shape[1]
# SPLIT TRAIN TEST DATA
ratio = 0.85
train_ratio = int(ratio * len(x_data))
x_train = x_data[0:train_ratio]
x_test = x_data[train_ratio:]
y_train = y_data[0:train_ratio]
y_test = y_data[train_ratio:]
# GENERATE RANDOM SEQUENCES
batch_size = 32
sequence_length = 24
EPOCHS = 50
def batch_generator(x_train, y_train, batch_size, sequence_length, num_x_signals, num_y_signals, num_train):
while True:
x_shape = (batch_size, sequence_length, num_x_signals)
x_batch = np.zeros(shape = x_shape).astype(np.float32)
y_shape = (batch_size, sequence_length, num_y_signals)
y_batch = np.zeros(shape = y_shape).astype(np.float32)
for i in range(batch_size):
idx = np.random.randint(num_train - sequence_length)
x_batch[i] = x_train[idx:idx+sequence_length]
y_batch[i] = y_train[idx:idx+sequence_length]
yield (x_batch, y_batch)
generator = batch_generator(x_train, y_train, batch_size, sequence_length, num_x_signals, num_y_signals, train_ratio)
xtrain, ytrain = next(generator)
xtest, ytest = (np.expand_dims(x_test, axis=0),
np.expand_dims(y_test, axis=0))
# LSTM MODEL
model = Sequential()
model.add(LSTM(32, input_shape = (None, num_x_signals,), return_sequences = True))
model.add(Dropout(0.2))
model.add(BatchNormalization())
model.add(LSTM(128, return_sequences = True))
model.add(Dropout(0.15))
model.add(BatchNormalization())
model.add(LSTM(128))
model.add(Dropout(0.18))
model.add(BatchNormalization())
model.add(Dense(32, activation = 'relu'))
model.add(Dropout(0.2))
model.add(Dense(1, activation = 'softmax'))
opt = tf.keras.optimizers.Adam(lr = 0.001, decay = 1e-6)
model.compile(
loss = 'sparse_categorical_crossentropy',
optimizer = opt,
metrics = ['accuracy']
)
name_of_file = f"{to_predict}-{sequence_length}-{future_predict}-{int(time.time())}"
tensorboard = TensorBoard(log_dir = "logs/{}".format(name_of_file))
filepath = "LSTM_Final-{epoch:02d}-{val_acc:.3f}"
checkpoint = ModelCheckpoint("models/{}.model".format(filepath, monitor = 'val_acc', verbose = 1, save_best_only = True, mode = 'max')) # saves only the best ones
history = model.fit(
xtrain, ytrain,
epochs = EPOCHS,
validation_data = (xtest, ytest),
callbacks = [tensorboard, checkpoint]
)
score = model.evaluate(xtest, ytest, verbose = 0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
model.save("models/{}".format(name_of_file))
I found this issue had to do with the loss function specified.
My code:
import tensorflow as tf
from tensorflow import keras
model = tf.keras.Sequential([
keras.layers.Dense(64, activation=tf.nn.relu, input_shape=[3]),
keras.layers.Dense(64, activation=tf.nn.relu),
keras.layers.Dense(1)
])
#I changed the loss function from 'sparse_categorical_crossentropy' to 'mean_squared error'
model.compile(optimizer='adam',loss='mean_squared_error',metrics=['accuracy'])
X = train_dataset.to_numpy()
y = train_labels.to_numpy()
model.fit(X,y, epochs=5)
X shape was (920,3) and dtype = float64
y shape was (920,1) and dtype = float64
My problem was in the model.fit method. I took the 'sparse_categorical_crossentropy' function from an image recognition example and what I was trying here is a neural network for house prices prediction.
I have developed a keras model and trying to fit my data into the model. Here is the code for model.fit
def train(run_name, start_epoch, stop_epoch, img_w):
# Input Parameters
img_h = 64
words_per_epoch = 300
val_split = 0.2
val_words = int(words_per_epoch * (val_split))
# Network parameters
conv_filters = 16
kernel_size = (3, 3)
pool_size = 2
time_dense_size = 32
rnn_size = 256
minibatch_size = 32
if K.image_data_format() == 'channels_first':
input_shape = (1, img_w, img_h)
else:
input_shape = (img_w, img_h, 1)
act = 'relu'
input_data = Input(name='the_input', shape=input_shape, dtype='float32')
inner = Conv2D(conv_filters, kernel_size, padding='same',
activation=act, kernel_initializer='he_normal',
name='conv1')(input_data)
inner = MaxPooling2D(pool_size=(pool_size, pool_size), name='max1')(inner)
inner = Conv2D(conv_filters, kernel_size, padding='same',
activation=act, kernel_initializer='he_normal',
name='conv2')(inner)
inner = MaxPooling2D(pool_size=(pool_size, pool_size), name='max2')(inner)
conv_to_rnn_dims = (img_w // (pool_size ** 2), (img_h // (pool_size ** 2)) * conv_filters)
inner = Reshape(target_shape=conv_to_rnn_dims, name='reshape')(inner)
# cuts down input size going into RNN:
inner = Dense(time_dense_size, activation=act, name='dense1')(inner)
# Two layers of bidirectional GRUs
# GRU seems to work as well, if not better than LSTM:
gru_1 = GRU(rnn_size, return_sequences=True, kernel_initializer='he_normal', name='gru1')(inner)
gru_1b = GRU(rnn_size, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', name='gru1_b')(inner)
gru1_merged = add([gru_1, gru_1b])
gru_2 = GRU(rnn_size, return_sequences=True, kernel_initializer='he_normal', name='gru2')(gru1_merged)
gru_2b = GRU(rnn_size, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', name='gru2_b')(gru1_merged)
# transforms RNN output to character activations:
# print("Output Size",img_gen.get_output_size())
inner = Dense(47, kernel_initializer='he_normal',
name='dense2')(concatenate([gru_2, gru_2b]))
y_pred = Activation('softmax', name='softmax')(inner)
# Model(inputs=input_data, outputs=y_pred).summary()
labels = Input(name='the_labels', shape=[10], dtype='float32')
input_length = Input(name='input_length', shape=[1], dtype='int64')
label_length = Input(name='label_length', shape=[1], dtype='int64')
# Keras doesn't currently support loss funcs with extra parameters
# so CTC loss is implemented in a lambda layer
loss_out = Lambda(ctc_lambda_func, output_shape=(1,), name='ctc')([y_pred, labels, input_length, label_length])
# clipnorm seems to speeds up convergence
sgd = SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5)
model = Model(inputs=[input_data, labels, input_length, label_length], outputs=loss_out)
# the loss calc occurs elsewhere, so use a dummy lambda func for the loss
model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=sgd)
test_func = K.function([input_data], [y_pred])
# viz_cb = VizCallback(run_name, test_func, img_gen.next_val())
(X_train, y_train, train_input_length, train_labels_length), (X_test, y_test, test_input_length, test_labels_length) = dataset_load('./OCR_BanglaData.pkl.gz')
print(y_train[0])
X_train = X_train.reshape(X_train.shape[0], 128,64,1)
X_test = X_test.reshape(X_test.shape[0], 128,64,1)
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
model.fit((np.array(X_train), np.array(y_train),np.array(train_input_length), np.array(train_labels_length)), batch_size=32, epochs=120, verbose=1,validation_data=[np.array(X_test), np.array(y_test),np.array(test_input_length), np.array(test_labels_length)])
After this I am getting this error
TypeError: Error when checking model input: data should be a Numpy array, or list/dict of Numpy arrays.
I tried to print the data types of each array. The result is
<type 'numpy.ndarray'>
But still I am getting this error. Is there any specific reason for it?
I am using tensorflow model as my backend of keras.
Here you have 4 inputs:
model = Model(inputs=[input_data, labels, input_length, label_length], outputs=loss_out)
Each of the four inputs must be a numpy array, and the fit method wants them in a list (you're using a tuple):
model.fit([X_train, y_train,train_input_length, train_labels_length],...)
And you're missing the outputs in the fit method. Something must match what you defined as loss_out when creating the model.