How to Merge two CNN models?

Question

I have 1D-CNN model and 2D-CNN model and want to merge them as mention in this paper , How can i merge them ? any help will appreciate , Thank you very much!

from keras import Sequential, Model
from keras.layers.core import Dense, Activation
from keras.layers.convolutional import Conv2D , Conv1D
from keras.layers import Conv2D, Conv1D,MaxPooling2D, Reshape, Concatenate, Dropout , MaxPooling1D
from keras.layers.merge import concatenate
from keras.layers import Dense, Input

model_1D = Sequential()
# 1
model_1D.add(Conv1D(32, kernel_size= 5 , strides=1, activation='relu' , input_shape = (7380, 128000)))
model_1D.add(MaxPooling1D(pool_size= 4, strides=4))
# 2 
model_1D.add(Conv1D(32, kernel_size= 5 , strides=1 , activation='relu'))
model_1D.add(MaxPooling1D(pool_size= 4, strides=4))
# 3
model_1D.add(Conv1D(64, kernel_size= 5 , strides=1 , activation='relu'))
model_1D.add(MaxPooling1D(pool_size= 4, strides=4))
# 4 
model_1D.add(Conv1D(64, kernel_size= 5 , strides=1 , activation='relu'))
model_1D.add(MaxPooling1D(pool_size= 2, strides=2))
# 5
model_1D.add(Conv1D(128, kernel_size= 5 , strides= 1 , activation='relu'))
model_1D.add(MaxPooling1D(pool_size= 2, strides= 2))
# 6
model_1D.add(Conv1D(128, kernel_size= 5 , strides= 1 , activation='relu'))
model_1D.add(MaxPooling1D(pool_size= 2, strides= 2))
model_1D.add(Dense(9 , activation='relu'))
#model_1D.summary()
# ----------------------- 2D CNN ----------------------
model_2D = Sequential()
model_2D.add(Conv2D(32, kernel_size=(3, 3) , strides=(1,1), activation='relu' , input_shape = (7380, 128, 251)))
model_2D.add(Conv2D(32, kernel_size=(3, 3) , strides=(1,1), activation='relu'))
model_2D.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model_2D.add(Conv2D(32, kernel_size=(3, 3) , strides=(1,1), activation='relu'))
model_2D.add(Conv2D(32, kernel_size=(3, 3) , strides=(1,1), activation='relu'))
model_2D.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model_2D.add(Dense(9 , activation='relu'))
model_2D.summary()

Answer 1

You want to build one model which consists of two branches, not two models, just like the paper says. Both branches need to be merged together using the Concatenate() layer. Also one other thing that was missing from your code were 'Flatten()' layers which must be insterted before the last Dense() layer of each branch. Here is what I propose:

from keras import Model
from keras.layers.core import Dense, Activation
from keras.layers import Conv2D, Conv1D, MaxPooling2D, Reshape, Concatenate, Dropout , MaxPooling1D, Flatten
from keras.layers import Dense, Input

model_1D = Input((7380, 128000))
# 1
model_1D = Conv1D(32, kernel_size= 5 , strides=1, activation='relu')(model_1D)
model_1D = MaxPooling1D(pool_size= 4, strides=4)(model_1D)
# 2
model_1D = Conv1D(32, kernel_size= 5 , strides=1 , activation='relu')(model_1D)
model_1D = MaxPooling1D(pool_size= 4, strides=4)(model_1D)
# 3
model_1D = Conv1D(64, kernel_size= 5 , strides=1 , activation='relu')(model_1D)
model_1D = MaxPooling1D(pool_size= 4, strides=4)(model_1D)
# 4
model_1D = Conv1D(64, kernel_size= 5 , strides=1 , activation='relu')(model_1D)
model_1D = MaxPooling1D(pool_size= 2, strides=2)(model_1D)
# 5
model_1D = Conv1D(128, kernel_size= 5 , strides= 1 , activation='relu')(model_1D)
model_1D = MaxPooling1D(pool_size= 2, strides= 2)(model_1D)
# 6
model_1D = Conv1D(128, kernel_size= 5 , strides= 1 , activation='relu')(model_1D)
model_1D = MaxPooling1D(pool_size= 2, strides= 2)(model_1D)
model_1D = Flatten()(model_1D)
model_1D = Dense(9 , activation='relu')(model_1D)
# ----------------------- 2D CNN ----------------------
model_2D = Input((7380, 128, 251))
model_2D = Conv2D(32, kernel_size=(3, 3) , strides=(1,1), activation='relu')(model_2D)
model_2D = Conv2D(32, kernel_size=(3, 3) , strides=(1,1), activation='relu')(model_2D)
model_2D = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(model_2D)
model_2D = Conv2D(32, kernel_size=(3, 3) , strides=(1,1), activation='relu')(model_2D)
model_2D = Conv2D(32, kernel_size=(3, 3) , strides=(1,1), activation='relu')(model_2D)
model_2D = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(model_2D)
model_2D = Flatten()(model_2D)
model_2D = Dense(9 , activation='relu')(model_2D)
merged = Concatenate()([model_1D, model_2D])
output = Dense(7, activation='softmax')(merged)

model_final = Model(inputs=[in_1D, in_2D], outputs=[output])

Note that input layers must exist explicitly, so they can be bound into the model's inputs. Compile and visualize the final model to make sure the architecture is correct before training:

model_final.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
from keras.utils import plot_model
plot_model(model_final, to_file='model_final.png')

In your own work you can use any other loss functions, optimizer etc.