so i am currently implementing the model in the following paper https://openaccess.thecvf.com/content_cvpr_2018/papers/Oh_Fast_Video_Object_CVPR_2018_paper.pdf
And as the following model show they used 2 resnet50 inside the model
Model's Image
Labeled as the siamese encoders
I used the resnet50 model provided by Keras with following code :
input_shape = (480,854,4)
inputlayer_Q = Input(shape=input_shape, name="inputlayer_Q")
convlayer_Q = Conv2D(filters= 3,kernel_size = (3,3),padding = 'same')(inputlayer_Q)
model_Q = tf.keras.applications.resnet50.ResNet50(
input_shape=(
convlayer_Q.shape[1],convlayer_Q.shape[2],convlayer_Q.shape[3]),
include_top=False,
weights='imagenet'
)
They then took 3 skip connections from layers inside the resnet model, I tried to takes the skip connections by using the following line
res2_skip = model_Q.layers[38].output
res3_skip = model_Q.layers[80].output
res4_skip = model_Q.layers[142].output
But when I use it in later in the model and try to run it give me Graph disconnected.
So is there any way to make skip connections/modify models provided by Keras ?
Try this:
input_shape = (480,854,4)
# Target Stream = Q
inputlayer_Q = Input(shape=input_shape, name="inputlayer_Q")
# Refrence Stream = M
inputlayer_M = Input(shape=input_shape,name="inputlayer_M")
convlayer_Q = Conv2D(filters= 3,kernel_size = (3,3),padding = 'same')(inputlayer_Q)
convlayer_M = Conv2D(filters= 3,kernel_size = (3,3),padding = 'same')(inputlayer_M)
model_Q = tf.keras.applications.resnet50.ResNet50(
input_shape=(convlayer_Q.shape[1],convlayer_Q.shape[2],convlayer_Q.shape[3]), include_top=False, weights='imagenet'
)
model_Q._name ="resnet50_Q"
model_M = tf.keras.applications.resnet50.ResNet50(
input_shape=(convlayer_M.shape[1],convlayer_M.shape[2],convlayer_M.shape[3]), include_top=False, weights='imagenet'
)
model_M._name ="resnet50_M"
for model in [model_Q, model_M]:
for layer in model.layers:
old_name = layer.name
layer._name = f"{model.name}_{old_name}"
print(layer._name)
encoder_Q = tf.keras.Model(inputs=model_Q.inputs, outputs=model_Q.output,name ="encoder_Q" )
encoder_M = tf.keras.Model(inputs=model_M.inputs, outputs=model_M.output,name ="encoder_M" )
concatenate = Concatenate(axis=0,name ="Concatenate")([encoder_Q.output, encoder_M.output])
global_layer = GlobalConvBlock(concatenate)
res2_skip = encoder_Q.layers[38].output
res2_skip = ZeroPadding2D(padding=(0,1), data_format=None)(res2_skip)
res3_skip = encoder_Q.layers[80].output
res3_skip = ZeroPadding2D(padding=((0,0),(0,1)), data_format=None)(res3_skip)
res4_skip = encoder_Q.layers[142].output
ref1_16 = refineblock(res4_skip,global_layer,"ref1_16")
ref1_8 = refineblock(res3_skip,ref1_16,"ref1_8")
ref1_4 = refineblock(res2_skip,ref1_8,"ref1_4")
outconv = Conv2D(filters= 2,kernel_size = (3,3)) (ref1_4)
outconv1 = ZeroPadding2D(padding=((1,1),(0,0)), data_format=None)(outconv)
output = Softmax()(outconv1)
main_model = tf.keras.Model(inputs=[encoder_Q.inputs, encoder_M.inputs],outputs=output, name ="main model" )
Related
I have already trained a model with this structure using first order optimization. The code is below:
def build_model():
input_layer1 = Input((dataX.shape[1],))
input_layer2 = Input((dataT1.shape[1],))
input_layer3 = Input((dataT2.shape[1],))
input_layer4 = Input((dataT3.shape[1],))
input_layer5 = Input((dataT4.shape[1],))
#continuous layers
hidden_1 = K.layers.Dense(64, activation='tanh')(input_layer1)
hidden_2 = K.layers.Dense(64, activation='tanh')(hidden_1)
hidden_3 = K.layers.Dense(64, activation='tanh')(hidden_2)
hidden_4 = K.layers.Dense(64, activation='tanh')(hidden_3)
#categorical layers and merging
hidden_5 = K.layers.Dense(64, activation='tanh')(input_layer2)
hidden_6 = K.layers.Dense(64, activation='tanh')(input_layer3)
hidden_7 = K.layers.Dense(64, activation='tanh')(input_layer4)
hidden_8 = K.layers.Dense(64, activation='tanh')(input_layer5)
merged1 = merge.concatenate([hidden_5, hidden_6])
merged2 = merge.concatenate([hidden_7, hidden_8])
merged3 = merge.concatenate([merged1, merged2])
hidden_9 = K.layers.Dense(64, activation='tanh')(merged3)
hidden_10 = K.layers.Dense(32, activation='tanh')(hidden_9)
hidden_11 = K.layers.Dense(16, activation='tanh')(hidden_10)
hidden_12 = K.layers.Dense(8, activation='tanh')(hidden_11)
hidden_13 = K.layers.Dense(4, activation='tanh')(hidden_12)
hidden_14 = K.layers.Dense(2, activation='tanh',name='2D')(hidden_13)
#completely merged kayers
merged_layers = merge.concatenate([hidden_4, hidden_14])
merged=K.layers.Dense(64, activation='tanh')(merged_layers)
merged1=K.layers.Dense(64, activation='tanh')(merged)
merged2=K.layers.Dense(64, activation='tanh')(merged1)
merged3=K.layers.Dense(64, activation='tanh')(merged2)
output=K.layers.Dense(1, activation='tanh')(merged3)
model = Model(inputs=[input_layer1,input_layer2,input_layer3,input_layer4,input_layer5], outputs=output)
model.compile(optimizer='rmsprop', loss='mse', metrics=['mae'])
return model
The optimizers offered by TensorFlow are all first order. I want to use second-order optimization and was wondering how I would do this? All the examples I have looked at describe a way for sequential models with second order optimization such as L-BFGS but not functional models.
Team I'm trying to classify LSTM on donors choose dataset with multiple inputs, But
I'm getting the following error
TypeError: len is not well defined for symbolic Tensors. (sub_4:0) Please call x.shape rather than len(x) for shape information.
The is code for my architecture as below
# Create model
# tf.keras.backend.clear_session()
keras.backend.clear_session()
input1 = Input(shape=(pad_essay.shape[1],),name="essay_input")
embedding_layer_text = Embedding(vocab_size_text,300, weights=[embedding_matrix],input_length=max_length_text,trainable=False)(input1)
lstm1 = LSTM(300,activation='relu',return_sequences=True,kernel_constraint=keras.initializers.he_normal(seed=0))(embedding_layer_text)
flatten1 = Flatten()(lstm1)
input2 = Input(shape=(1,),name = "school_state")
embedding_layer_school_state = Embedding(unique_school_state-1,20,input_length=1)(input2)
flatten2 = Flatten()(embedding_layer_school_state)
input3 = Input(shape=(1,),name ="project_grade_category")
embedding_layer_grade = Embedding(unique_project_grade_category-1,20,input_length=1)(input3)
flatten3 = Flatten()(embedding_layer_grade)
input4 = Input(shape=(1,),name='categories')
embedding_layer_categories = Embedding(unique_project_subject_categories-1,10,input_length=1)(input4)
flatten4 = Flatten()(embedding_layer_categories)
input5 = Input(shape=(1,),name='sub_categories')
embedding_layer_sub_categories = Embedding(unique_project_subject_subcategories-1,30,input_length=1)(input5)
flatten5 = Flatten()(embedding_layer_sub_categories)
input6 = Input(shape=(1,),name = 'teacher_prefix')
embedding_layer_teacher_prefix = Embedding(unique_teacher_prefix,10,input_length=1)(input6)
flatten6 = Flatten()(embedding_layer_teacher_prefix)
input7 = Input(shape=(numerical_df.shape[1],1),name ='other_numerical_input')
dense1 = Dense(64,activation='relu',kernel_initializer=keras.initializers.he_normal(seed=0))(input7)
flatten7 = Flatten()(dense1)
concate_layer = concatenate(inputs=[flatten1,flatten2,flatten3,flatten4,flatten5,flatten6,flatten7],name="concat",axis=-1)
dense_layer1_after_concat = Dense(128,activation='relu',kernel_initializer=keras.initializers.he_normal(seed=0))(concate_layer)
drop_out1_after_concat = Dropout(0.3)(dense_layer1_after_concat)
dense_layer2_after_concat = Dense(64,activation='relu',kernel_initializer=keras.initializers.he_normal(seed=0))(drop_out1_after_concat)
drop_out2_after_concat = Dropout(0.3)(dense_layer2_after_concat)
dense_layer3_after_concat = Dense(32,activation="relu",kernel_initializer=keras.initializers.he_normal(seed=0))(drop_out2_after_concat)
output_layer = Dense(units=1,activation='sigmoid',kernel_initializer=keras.initializers.glorot_uniform(seed=0),name="output")(dense_layer3_after_concat)
first_model = Model(inputs = [input1,input2,input3,input4,input5,input6,input7],outputs=output_layer)
opt = Adam(lr=1e-3, decay=1e-3 / 200)
first_model.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy'])
# first_model.compile(loss='binary_crossentropy',optimizer=opt,metrics=["accuracy"])
first_model.fit([pad_essay,X_train_school_state_le,X_train_project_grade_category_le,X_train_project_subject_categories_le,X_train_project_project_subject_subcategories_le,X_train_teacher_prefix_le,numerical_df1],y_train,validation_data=([pad_essay_test,X_test_school_state_le,X_test_project_grade_category_le,X_test_project_subject_categories_le,X_test_project_subject_subcategories_le,X_test_teacher_prefix_le,numerical_df_test1],y_test),epochs=10,batch_size=64,callbacks=[roc])
I have done custom encoding for categorical variable
and use predefined model(word2vec) for text data
The sample custom encoding as below
le = LabelEncoder()
unique_school_state = len(X_train['school_state'].unique())+1
X_train_school_state_le = le.fit_transform(X_train['school_state'].values)
le_dict_school_state = dict(zip(le.classes_, le.transform(le.classes_)))
X_test_school_state_le= X_test['school_state'].apply(lambda x: le_dict_school_state.get(x,unique_school_state))
I am trying to access the weights of a Keras layer and use the weight values themselves as the input to a different layer.
Here's a rough outline of what I'm hoping to achieve:
def generate_myModel(SEQUENCE_LENGTH, FILT_NUM, FILT_SIZE):
ip = keras.layers.Input(shape = (SEQUENCE_LENGTH,1))
conv_layer = keras.layers.Conv1D(filters = FILT_NUM, kernel_size = FILT_SIZE)
y = conv_layer(ip)
y = keras.layers.GlobalMaxPooling1D()(y)
out_y = keras.layers.Dense(units = 1, activation = 'linear')(y)
# Acquire the actual weights from the previous convolution layers
w1 = <WEIGHTS FROM conv_layer - THIS IS THE PART IN QUESTION>
out_w1 = keras.layers.Lambda( lambda x: K.std(x)/K.abs(K.mean(x)) )(w1)
myModel = keras.models.Model(inputs = ip, outputs = [out_y, out_w1])
return myModel
I'm aware that, once you have an instantiated model, you can use model.layers[i].get_weights(), but I'd like to be able to do this inside the actual architecture of the model.
Is this possible?
EDIT------------------------------------
Attempting the solution from the comments, I add layer.get_weights() into the model's architecture as so:
def generate_myModel(SEQUENCE_LENGTH, FILT_NUM, FILT_SIZE):
ip = keras.layers.Input(shape = (SEQUENCE_LENGTH,1))
conv_layer = keras.layers.Conv1D(filters = FILT_NUM, kernel_size = FILT_SIZE)
y = conv_layer(ip)
# Acquire the actual weights from the previous convolution layers
w1 = K.constant(conv_layer.get_weights()) # layer.get_weights returns a Numpy
# array, I need a Keras Tensor -
# so I use K.constant()
y = keras.layers.GlobalMaxPooling1D()(y)
out_y = keras.layers.Dense(units = 1, activation = 'linear')(y)
out_w1 = keras.layers.Lambda( lambda x: K.std(x)/K.abs(K.mean(x)) )(w1)
myModel = keras.models.Model(inputs = ip, outputs = [out_y, out_w1])
return myModel
but this leaves me with the following error:
AttributeError: 'NoneType' object has no attribute '_inbound_nodes'
Any guidance would be greatly appreciated!
I have a dataset of N folder(N ID), each N ID folder have M folder inside, and each M folder have 8 images inside it. I want to train the dataset with 2D-CNN. My model contain 8 CNNs each one take one of the M folder image, after ending the first folder of that ID, the model take the next folder with 8 images and each image goes to one of the 8 models and so on. finally I concatenate the output of the 8 models, but I faced a problem when I want to concatenate all dataset. How can I concatenate the output of the first 8 models, with the output of the second 8 models and so on till the dataset ended. my model design is as the follwing images:
my python code is as following:
model_out = []
input_list = []
model_list = []
for fold_Path in listing:
image_fold = os.listdir(ID_Paths + "\\" + fold_Path)
for file in image_fold:
segments = os.listdir(ID_Paths + "\\" + fold_Path + "\\" + file)
segments_list = []
input_list = []
output_list = []
model_out = []
for seg in segments:
im = (ID_Paths + "\\" + fold_Path + "\\" + file + "\\" + seg)
image = cv2.imread(im)
image = cv2.resize(image, (60, 60))
segments_list.append(image)
if len(segments_list) == 8:
seg1 = Input(shape=segments_list[0].shape, name="seg1")
input_list.append(seg1)
conv0_1 = Conv2D(32, (3, 3), padding="same")(seg1)
act0_1 = Activation("relu")(conv0_1)
batch0_1 = BatchNormalization(axis=-1)(act0_1)
pool0_1 = MaxPooling2D(pool_size=(2, 2))(batch0_1)
drop0_1 = Dropout(0.25)(pool0_1)
conv0_2 = Conv2D(64, (3, 3), padding="same")(drop0_1)
act0_2 = Activation("relu")(conv0_2)
batch0_2 = BatchNormalization(axis=-1)(act0_2)
pool0_2 = MaxPooling2D(pool_size=(2, 2))(batch0_2)
drop0_2 = Dropout(0.25)(pool0_2)
out1 = Flatten()(drop0_2)
output_list.append(out1)
# the same design until model 8
.
.
.
seg8 = Input(shape=segments_list[7].shape, name="seg8")
input_list.append(seg8)
conv7_1 = Conv2D(32, (3, 3), padding="same")(seg8)
act7_1 = Activation("relu")(conv7_1)
batch7_1 = BatchNormalization(axis=-1)(act7_1)
pool7_1 = MaxPooling2D(pool_size=(2, 2))(batch7_1)
drop7_1 = Dropout(0.25)(pool7_1)
conv7_2 = Conv2D(64, (3, 3), padding="same")(drop7_1)
act7_2 = Activation("relu")(conv7_2)
batch7_2 = BatchNormalization(axis=-1)(act7_2)
pool7_2 = MaxPooling2D(pool_size=(2, 2))(batch7_2)
drop7_2 = Dropout(0.25)(pool7_2)
out8 = Flatten()(drop7_2)
output_list.append(out8)
# -----------Now Concatenation of 8 models will be start-----------------------------------------------------------------------------
merge = Concatenate()(output_list)
print("Concatenation Ended...Dense will be done...")
den1 = Dense(128)(merge)
act = Activation("relu")(den1)
bat = BatchNormalization()(act)
drop = Dropout(0.5)(bat)
model_out.append(drop)
else:
continue
small_model = Model(inputs=input_list, outputs=model_out)
model_list.append(small_model)
print("Concatenation done")
segments_list = []
input_list = []
output_list = []
model_out = []
# it is OK till here, after this step I don't know how can I concatenate the output of each concatenated result
den2 = Dense(128)(model_list) # the error in this line
act2 = Activation("relu")(den2)
bat2 = BatchNormalization()(act2)
drop2 = Dropout(0.5)(bat2)
# softmax classifier
print("Classification will be start")
final_out1 = Dense(classes)(drop2)
final_out = Activation('softmax')(final_out1)
#inp = Input(shape=den2.shape)
#big_model = Model(inputs=inp, outputs=final_out)
final_out.compile(loss="categorical_crossentropy", optimizer= opt, metrics=["accuracy"])
final_out.fit_generator(aug.flow(trainX, trainY, batch_size=BS),validation_data=(testX, testY),steps_per_epoch=len(trainX) // BS, epochs=EPOCHS, verbose=1)
When I run the program, it gives me the following error:
ValueError: Layer dense_66 was called with an input that isn't a symbolic tensor.
Can anyone please help me. How can I concatenate, compile, train the all dataset. any hint may be helpful, thanks.
That is because you are passing a list of model objects model_list which are not tensors, they wrap computation graphs that given inputs produce tensors. Instead you should collect the tensor outputs, something along the lines of:
#...
model_ins.append(seg1)
# ...
model_outs.append(drop)
# ...
all_model_outs = Concatenate(model_outs)
flat_model_outs = Flatten()(all_model_outs)
den2 = Dense(128)(flat_model_outs) # the error in this line
# ...
big_model= Model(model_ins, final_out)
big_model.compile(loss="categorical_crossentropy", optimizer= opt, metrics=["accuracy"])
big_model.fit_generator(aug.flow(trainX, trainY, batch_size=BS),validation_data=(testX, testY),steps_per_epoch=len(trainX) // BS, epochs=EPOCHS, verbose=1)
The idea is you can take any input and output computation of a larger graph and convert to a model to train it. Here, the big model is all the inputs to the final output you compute that will train all the smaller models together. You can still use the smaller models to predict individually later.
Please help. I have been trying to use the DeepBeliefTrainer with a simple Deep Neural Network to no success. I am using the DeepBeliefTrainer available here: http://tinyurl.com/jeexh5g
I am getting the following error:
File "...\pybrain\structure\modules\module.py", line 104, in activate
assert len(self.inputbuffer[self.offset]) == len(inpt), str((len(self.inputbuffer[self.offset]), len(inpt)))
AssertionError: (2, 1)
Below is my code:
ds = SupervisedDataSet(2, 1)
ds.addSample([0,0],[0])
ds.addSample([0,1],[1])
ds.addSample([1,0],[1])
ds.addSample([1,1],[0])
n = FeedForwardNetwork()
no_neurons_hidden_layers = int(round(0.6 * ds.indim))
# creating layers
bias = BiasUnit(name='bias')
inLayer = LinearLayer(ds.indim,name='visible')
hiddenLayer1 = TanhLayer(no_neurons_hidden_layers,name='h1')
hiddenLayer2 = TanhLayer(no_neurons_hidden_layers,name='h2')
hiddenLayer3 = TanhLayer(no_neurons_hidden_layers,name='h3')
outLayer = SoftmaxLayer(ds.outdim,name='out')
# adding layers to modules
n.addInputModule(inLayer)
n.addModule(bias)
n.addModule(hiddenLayer1)
n.addModule(hiddenLayer2)
n.addModule(hiddenLayer3)
n.addOutputModule(outLayer)
# connecting layers to each other
inLayer_to_hidden1 = FullConnection(inLayer, hiddenLayer1)
bias_to_hidden1 = FullConnection(bias, hiddenLayer1)
hidden1_to_hidden2 = FullConnection(hiddenLayer1,hiddenLayer2)
hidden2_to_hidden3 = FullConnection(hiddenLayer2,hiddenLayer3)
hidden3_to_outLayer = FullConnection(hiddenLayer3,outLayer)
bias_to_outLayer = FullConnection(bias, outLayer)
#add connections
n.addConnection(inLayer_to_hidden1)
n.addConnection(bias_to_hidden1)
n.addConnection(hidden1_to_hidden2)
n.addConnection(hidden2_to_hidden3)
n.addConnection(hidden3_to_outLayer)
n.addConnection(bias_to_outLayer)
#sort modules
n.sortModules()
#make a Deep Belief Network
cfg = RbmGibbsTrainerConfig()
cfg.maxIter = 3
dbn = DeepBeliefTrainer(n,ds,epochs=50,cfg=cfg, distribution='bernoulli')
dbn.train()
Many thanks for any help.