I'm making a CNN model to categorise MRI brain scans into Alzheimer's and Healthy groups.
It currently seems as though it's overfitting, and we have tried a lot of tricks in the book to fix this issue.
One method I'd now like to try - but have no experience in - is using multiple images of the brain scan per data point.
For example:
I download the brain scans in .nii format and conver them into a series of png files. Each png file is of the same scan at different points in time. With my current model, I have simply deleted all but one of these images per brain scan, making my model easier to work with.
What I'd now like to try is to use multiple images - say 5 to begin with - for each sample in my dataset.
How would I go about achieving this by altering my current model?
I do not want to use the data augmentation feature of the ImageDataGenerator, I'd like to use existing files which I have stored on my computer.
I'd appreciate any and all responses as the current model is sitting at ~80% training accuracy and 55% val accuracy.
# Use ImageDataGenerator to create 3 lots of batches
train_batches = ImageDataGenerator(
rescale=1/255).flow_from_directory(directory=train_path,
target_size=(80,80), classes=['cn', 'ad'], batch_size=100,
color_mode="rgb")
valid_batches = ImageDataGenerator(
rescale=1/255).flow_from_directory(directory=valid_path,
target_size=(80,80), classes=['cn', 'ad'], batch_size=100,
color_mode="rgb")
# test_batches = ImageDataGenerator(
# rescale=1/255).flow_from_directory(directory=test_path,
# target_size=(224,224), classes=['cn', 'ad'], batch_size=10,
# color_mode="rgb")
imgs, labels = next(train_batches)
# Test to see normalisation has occurred properly
print(imgs[1][8])
# Define method to plot MRIs
def plotImages(images_arr):
fig, axes = plt.subplots(1, 10, figsize=(20,20))
axes = axes.flatten()
for img, ax in zip( images_arr, axes):
ax.imshow(img)
ax.axis('off')
plt.tight_layout()
plt.show()
# Plot a sample of MRIs
plotImages(imgs)
# # Define the model
# # VGG16
# model = Sequential()
# model.add(Conv2D(input_shape=(160,160,3),filters=64,kernel_size=(3,3),padding="same", activation="relu"))
# model.add(Conv2D(filters=64,kernel_size=(3,3),padding="same", activation="relu"))
# model.add(MaxPool2D(pool_size=(2,2),strides=(2,2)))
# model.add(Conv2D(filters=128, kernel_size=(3,3), padding="same", activation="relu"))
# model.add(Conv2D(filters=128, kernel_size=(3,3), padding="same", activation="relu"))
# model.add(MaxPool2D(pool_size=(2,2),strides=(2,2)))
# model.add(Conv2D(filters=256, kernel_size=(3,3), padding="same", activation="relu"))
# model.add(Conv2D(filters=256, kernel_size=(3,3), padding="same", activation="relu"))
# model.add(Conv2D(filters=256, kernel_size=(3,3), padding="same", activation="relu"))
# model.add(MaxPool2D(pool_size=(2,2),strides=(2,2)))
# model.add(Conv2D(filters=512, kernel_size=(3,3), padding="same", activation="relu"))
# model.add(Conv2D(filters=512, kernel_size=(3,3), padding="same", activation="relu"))
# model.add(Conv2D(filters=512, kernel_size=(3,3), padding="same", activation="relu"))
# model.add(MaxPool2D(pool_size=(2,2),strides=(2,2)))
# model.add(Conv2D(filters=512, kernel_size=(3,3), padding="same", activation="relu"))
# model.add(Conv2D(filters=512, kernel_size=(3,3), padding="same", activation="relu"))
# model.add(Conv2D(filters=512, kernel_size=(3,3), padding="same", activation="relu"))
# model.add(MaxPool2D(pool_size=(2,2),strides=(2,2)))
# model.add(Flatten())
# model.add(Dense(units=1024,activation="relu"))
# model.add(Dense(units=128,activation="relu"))
# model.add(Dense(units=2, activation="softmax"))
# # Model from the paper
# model = Sequential([
# Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding = 'same', input_shape=(160,160,3)),
# Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding='same'),
# MaxPool2D(pool_size=(2, 2), strides=2),
# Flatten(),
# Dense(units=2, activation='softmax')
# ])
## Model from Dr Paul
# static_conv_layer=Conv2D(filters=16, kernel_size=(5, 5), activation='relu', padding = 'same')
#
# model = Sequential([
# Conv2D(filters=16, kernel_size=(5, 5), activation='relu', padding = 'same', input_shape=(32,32,3)),
# MaxPool2D(pool_size=(2, 2), strides=2),
# Dropout(0.1),
# static_conv_layer,
# MaxPool2D(pool_size=(2, 2), strides=2),
# Dropout(0.1),
# Flatten(),
# Dense(units=2, activation='softmax')
# ])
# This model hits around 75% train acc, 54% val acc
model = Sequential([
Conv2D(filters=16, kernel_size=(5, 5), activation='relu', padding = 'same', input_shape=(80,80,3)),
MaxPool2D(pool_size=(2, 2), strides=2),
# Dropout(0.1),
# Conv2D(filters=16, kernel_size=(3, 3), activation='relu', padding='same'),
# MaxPool2D(pool_size=(2, 2), strides=2),
# Conv2D(filters=16, kernel_size=(3, 3), activation='relu', padding='same'),
# MaxPool2D(pool_size=(2, 2), strides=2),
Flatten(),
Dense(units=2, activation='softmax')
])
# model = Sequential([
# Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding = 'same', input_shape=(160,160,3)),
# Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding='same'),
# MaxPool2D(pool_size=(2, 2), strides=2),
# Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding='same'),
# Flatten(),
# Dense(units=2, activation='softmax')
# ])
## Basic model with dropouts
# model = Sequential([
# Conv2D(filters=32, kernel_size=(3, 3), activation='relu', padding = 'same', input_shape=(224,224,3)),
# MaxPool2D(pool_size=(2, 2), strides=2),
# Dropout(0.1),
# Conv2D(filters=64, kernel_size=(3, 3), activation='relu', padding='same'),
# MaxPool2D(pool_size=(2, 2), strides=2),
# Dropout(0.2),
# Conv2D(filters=128, kernel_size=(3, 3), activation='relu', padding='same'),
# MaxPool2D(pool_size=(2, 2), strides=2),
# Dropout(0.3),
# Flatten(),
# Dense(units=1, activation='sigmoid')
# ])
# Summarise each layer of the model
print(model.summary())
# Compile and train the model
model.compile(optimizer=Adam(), loss='binary_crossentropy', metrics=['accuracy'])
model.fit(x=train_batches,
steps_per_epoch=len(train_batches),
validation_data=valid_batches,
validation_steps=len(valid_batches),
epochs=20,
verbose=1
)
In order to incorporate multiple images in your model now, you can take your multiple images for each data point, the easiest thing would be to average them and you can try that. But, a better approach will be to apply dimensionality reduction on the multiple images.
First, it will consolidate all the features for each person and also, if number of images is different for each person, then, this would ease your training.
It would be better to compute the features outside and then, plug them into your model.
For that consider, PCA first.
Related
I generated ConvLSTM2D model for forecasting some raster file, the actual value is between 20 until 50, but when I get the result it has range between 0.49 and 0.5. But, if I compare the test result with actual data, it has same pattern. I am a newbie here, so any respond will be appreciate a lot
This is my model below
`
seq = keras.Sequential(
[
keras.Input(shape=(None, 128, 128, 1)),
layers.ConvLSTM2D(filters=40, kernel_size=(3,3), padding='same', return_sequences=True),
layers.BatchNormalization(),
layers.ConvLSTM2D(filters=40, kernel_size=(3,3), padding='same', return_sequences=True),
layers.BatchNormalization(),
layers.ConvLSTM2D(filters=40, kernel_size=(3,3), padding='same', return_sequences=True),
layers.BatchNormalization(),
layers.ConvLSTM2D(filters=40, kernel_size=(3,3), padding='same', return_sequences=True),
layers.BatchNormalization(),
layers.Conv3D(filters=1, kernel_size=(3, 3, 3), activation='sigmoid', padding='same', data_format='channels_last'),
]
)
I expect the result has some criteria (about range value) with the validation/training data.
I am building a VGG image pipeline, and I am trying to input 2 consecutive frames from a video as follows:
datagen = ImageDataGenerator()
datagen.fit(X_train)
model = Sequential()
model.add(Conv2D(input_shape=(224, 224, 6), filters=64, kernel_size=(3, 3), padding='same', activation='relu', kernel_initializer='he_uniform', bias_initializer='zeros'))
model.add(Conv2D(filters=64, kernel_size=(3, 3), padding='same', activation='relu', kernel_initializer='he_uniform', bias_initializer='zeros'))
model.add(MaxPool2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Conv2D(filters=128, kernel_size=(3, 3), padding='same', activation='relu', kernel_initializer='he_uniform', bias_initializer='zeros'))
model.add(Conv2D(filters=128, kernel_size=(3, 3), padding='same', activation='relu', kernel_initializer='he_uniform', bias_initializer='zeros'))
model.add(MaxPool2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Conv2D(filters=256, kernel_size=(3, 3), padding='same', activation='relu', kernel_initializer='he_uniform', bias_initializer='zeros'))
model.add(Conv2D(filters=256, kernel_size=(3, 3), padding='same', activation='relu', kernel_initializer='he_uniform', bias_initializer='zeros'))
model.add(Conv2D(filters=256, kernel_size=(3, 3), padding='same', activation='relu', kernel_initializer='he_uniform', bias_initializer='zeros'))
model.add(MaxPool2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Conv2D(filters=512, kernel_size=(3, 3), padding='same', activation='relu', kernel_initializer='he_uniform', bias_initializer='zeros'))
model.add(Conv2D(filters=512, kernel_size=(3, 3), padding='same', activation='relu', kernel_initializer='he_uniform', bias_initializer='zeros'))
model.add(Conv2D(filters=512, kernel_size=(3, 3), padding='same', activation='relu', kernel_initializer='he_uniform', bias_initializer='zeros'))
model.add(MaxPool2D(pool_size=(2, 2), strides=(2, 2)))
model.add(GlobalAveragePooling2D())
model.add(Dense(units=4096, activation='relu', kernel_initializer='he_uniform', bias_initializer='zeros'))
model.add(Dropout(0.2))
model.add(Dense(units=4096, activation='relu', kernel_initializer='he_uniform', bias_initializer='zeros'))
model.add(Dropout(0.2))
model.add(Dense(units=1, activation='sigmoid'))
# opt = Adam(learning_rate=0.001)
opt = SGD(lr=0.01, momentum=0.3)
checkpoint = ModelCheckpoint(config.CLASH_PATH() + '/models/step_01.h5', monitor='binary_accuracy', verbose=1, save_best_only=True,
save_weights_only=False, mode='auto', period=1)
early = EarlyStopping(monitor='binary_accuracy', min_delta=0, patience=40, verbose=1, mode='auto')
model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['binary_accuracy'])
model.summary()
model.fit(datagen.flow(X_train, y_train, batch_size=32,
subset='training', ignore_class_split=True), validation_data=datagen.flow(X_train, y_train, batch_size=16,
subset='validation', ignore_class_split=True), steps_per_epoch=len(X_train) / 48,
epochs=1000, verbose=1,
callbacks=[checkpoint, early])
As you'll note in the 4th line of code, I am passing 224x224x6 which represents two stacked image frames of 224x224x3. This is necessary since I am using the ImageDataGenerator to pass my data.
Unfortunately I am getting the following error message:
NumpyArrayIterator is set to use the data format convention "channels_last" (channels on axis 3), i.e. expected either 1, 3, or 4 channels on axis 3. However, it was passed an array with shape (6666, 224, 224, 6) (6 channels).
From other reading on stackoverflow, I have seen that I can stack my frames using layers.concatenate, but how would I then modify my generator to keep the flow of frames in sync?
I am trying to use the following code to for a CNN to classify images.
essentially, this code takes in a directory with folders and images, and trains a CNN to classify them.
This is my code:
self.model.add(Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same', input_shape=(32, 32, 3)))
self.model.add(BatchNormalization())
self.model.add(Conv2D(32, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))
self.model.add(BatchNormalization())
self.model.add(MaxPooling2D((2, 2)))
self.model.add(Dropout(0.2))
self.model.add(Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))
self.model.add(BatchNormalization())
self.model.add(Conv2D(64, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))
self.model.add(BatchNormalization())
self.model.add(MaxPooling2D((2, 2)))
self.model.add(Dropout(0.3))
self.model.add(Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))
self.model.add(BatchNormalization())
self.model.add(Conv2D(128, (3, 3), activation='relu', kernel_initializer='he_uniform', padding='same'))
self.model.add(BatchNormalization())
self.model.add(MaxPooling2D((2, 2)))
self.model.add(Dropout(0.4))
self.model.add(Flatten())
self.model.add(Dense(128, activation='relu', kernel_initializer='he_uniform'))
self.model.add(BatchNormalization())
self.model.add(Dropout(0.5))
self.model.add(Dense(10, activation='softmax'))
def processModel(self, split, epochs=10):
datagen = keras.preprocessing.image.ImageDataGenerator(
rescale=1./255,
rotation_range=5,
zoom_range=(0.95, 0.95),
horizontal_flip=False,
vertical_flip=False,
data_format='channels_last',
validation_split=split,
)
train_generator = datagen.flow_from_directory(
directory=self.path,
color_mode='rgb',
class_mode='sparse',
shuffle=True,
subset='training',
seed=7
)
test_generator = datagen.flow_from_directory(
directory=self.path,
color_mode='rgb',
class_mode='sparse',
shuffle=True,
subset='validation',
seed=7
)
self.model.compile(
optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=["accuracy"]
)
self.model.fit(train_generator, epochs=epochs, verbose=2)
self.score = self.model.evaluate(test_generator)
I keep getting this error:
logits and labels must have the same first dimension, got logits shape [2048,10] and labels shape [32]
on line:
self.model.fit(train_generator, epochs=epochs, verbose=2)
does anyone know why this may be happening?
Without knowing your data and data preparation it is hard to reproduce, but I think the problem could be your loss function
self.model.compile(
optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=["accuracy"]
)
Try using categorical_crossentropy instead sparse_categorical_crossentropy:
self.model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=["accuracy"]
)
I am trying to train a convolutional neural net. Therefore I am using a datset of 646 images/license plates which contains 8 characters (0-9, A-Z; without letter 'O' and blank spaces, in total 36 possible characters). These are my training data X_train. Their shape is (646, 40, 200, 3) with color code 3. I resized them to the same shape.
I also have a dataset which contains the labels of this images, which I one-hot-encoded to a numpy array of shape (646, 8, 36). This data is my y_train data.
Now, I am trying to apply a Neural Network which looks like this:
The architecture is taken from this paper: https://ieeexplore.ieee.org/abstract/document/8078501
I excluded the batch normalization part, because this part is not the most interesting one for me. But I am very unsure regarding the top of the layer. That means the part after the last pooling layer beginning with model.add(Flatten())...
model = Sequential()
model.add(Conv2D(32, kernel_size=(3, 3), input_shape = (40, 200, 3), activation = "relu"))
model.add(Conv2D(32, kernel_size=(3, 3), activation = "relu"))
model.add(Conv2D(32, kernel_size=(3, 3), activation = "relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, kernel_size=(3, 3), activation = "relu"))
model.add(Conv2D(64, kernel_size=(3, 3), activation = "relu"))
model.add(Conv2D(64, kernel_size=(3, 3), activation = "relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(128, kernel_size=(3, 3), activation = "relu"))
model.add(Conv2D(128, kernel_size=(3, 3), activation = "relu"))
model.add(Conv2D(128, kernel_size=(3, 3), activation = "relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(16000, activation = "relu"))
model.add(Dense(128, activation = "relu"))
model.add(Dense(36, activation = "relu"))
model.add(Dense(8*36, activation="Softmax"))
model.add(keras.layers.Reshape((8, 36)))
Thank you very much in advance!
Assuming the image below matches your model architecture, the code can be used to create the model. Ensure you have some padding for the input images.
import tensorflow as tf
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.layers import Conv2D, Flatten, MaxPooling2D, Dense, Input, Reshape, Concatenate
def create_model(input_shape = (40, 200, 3)):
input_img = Input(shape=input_shape)
model = Conv2D(32, kernel_size=(3, 3), input_shape = (40, 200, 3), activation = "relu")(input_img)
model = Conv2D(32, kernel_size=(3, 3), padding="same", activation = "relu")(model)
model = Conv2D(32, kernel_size=(3, 3), padding="same", activation = "relu")(model)
model = MaxPooling2D(pool_size=(2, 2))(model)
model = Conv2D(64, kernel_size=(3, 3), padding="same", activation = "relu")(model)
model = Conv2D(64, kernel_size=(3, 3), padding="same", activation = "relu")(model)
model = Conv2D(64, kernel_size=(3, 3), padding="same", activation = "relu")(model)
model = MaxPooling2D(pool_size=(2, 2))(model)
model = Conv2D(128, kernel_size=(3, 3), padding="same", activation = "relu")(model)
model = Conv2D(128, kernel_size=(3, 3), padding="same", activation = "relu")(model)
model = Conv2D(128, kernel_size=(3, 3), padding="same", activation = "relu")(model)
model = MaxPooling2D(pool_size=(2, 2))(model)
backbone = Flatten()(model)
branches = []
for i in range(8):
branches.append(backbone)
branches[i] = Dense(16000, activation = "relu", name="branch_"+str(i)+"_Dense_16000")(branches[i])
branches[i] = Dense(128, activation = "relu", name="branch_"+str(i)+"_Dense_128")(branches[i])
branches[i] = Dense(36, activation = "softmax", name="branch_"+str(i)+"_output")(branches[i])
output = Concatenate(axis=1)(branches)
output = Reshape((8, 36))(output)
model = Model(input_img, output)
return model
Here is my discriminator architecture:
def build_discriminator(img_shape,embedding_shape):
model1 = Sequential()
model1.add(Conv2D(32, kernel_size=5, strides=2, input_shape=img_shape, padding="same"))
model1.add(LeakyReLU(alpha=0.2))
model1.add(Dropout(0.25))
model1.add(Conv2D(48, kernel_size=5, strides=2, padding="same"))
#model.add(ZeroPadding2D(padding=((0,1),(0,1))))
model1.add(BatchNormalization(momentum=0.8))
model1.add(LeakyReLU(alpha=0.2))
model1.add(Dropout(0.25))
model1.add(Conv2D(64, kernel_size=5, strides=2, padding="same"))
model1.add(BatchNormalization(momentum=0.8))
model1.add(LeakyReLU(alpha=0.2))
model1.add(Dropout(0.25))
model1.add(Conv2D(128, kernel_size=5, strides=2, padding="same"))
model1.add(BatchNormalization(momentum=0.8))
model1.add(LeakyReLU(alpha=0.2))
model1.add(Dropout(0.25))
model1.add(Conv2D(256, kernel_size=5, strides=2, padding="same"))
model1.add(BatchNormalization(momentum=0.8))
model1.add(LeakyReLU(alpha=0.2))
model1.add(Dropout(0.25))
model1.add(Flatten())
model1.add(Dense(200))
model2=Sequential()
model2.add(Dense(50, input_shape=embedding_shape))
model2.add(Dense(100))
model2.add(Dense(200))
model2.add(Flatten())
merged_model = Sequential()
merged_model.add(Merge([model1, model2], mode='concat'))
merged_model.add(Dense(1, activation='sigmoid', name='output_layer'))
#merged_model.compile(loss='binary_crossentropy', optimizer='adam',
#metrics=['accuracy'])
#model1.add(Dense(1, activation='sigmoid'))
merged_model.summary()
merged_model.input_shape
img = Input(shape=img_shape)
emb = Input(shape=embedding_shape)
validity = merged_model([img,emb])
return Model([img,emb],validity)
and here is the generator architecture:
def build_generator(latent_dim=484):
model = Sequential()
model.add(Dense(624 * 2 * 2, activation="relu", input_dim=latent_dim))
model.add(Reshape((2, 2, 624)))
model.add(UpSampling2D())
model.add(Conv2D(512, kernel_size=5, padding="same"))
model.add(BatchNormalization(momentum=0.8))
model.add(Activation("relu"))
model.add(UpSampling2D())
#4x4x512
model.add(Conv2D(256, kernel_size=5, padding="same"))
model.add(BatchNormalization(momentum=0.8))
model.add(Activation("relu"))
model.add(UpSampling2D())
#8x8x256
model.add(Conv2D(128, kernel_size=5, padding="same"))
model.add(BatchNormalization(momentum=0.8))
model.add(Activation("relu"))
model.add(UpSampling2D())
#16x16x128
model.add(Conv2D(64, kernel_size=5, padding="same"))
model.add(BatchNormalization(momentum=0.8))
model.add(Activation("relu"))
model.add(UpSampling2D())
#32x32x64
model.add(Conv2D(32, kernel_size=5, padding="same"))
model.add(BatchNormalization(momentum=0.8))
model.add(Activation("relu"))
model.add(UpSampling2D())
#64x64x32
model.add(Conv2D(3, kernel_size=5, padding="same"))
model.add(Activation("tanh"))
#128x128x3
noise = Input(shape=(latent_dim,))
img = model(noise)
return Model(noise, img)
and here is how I am making the GAN network:
optimizer = Adam(0.0004, 0.5)
discriminator=build_discriminator((128,128,3),(1,128,3))
discriminator.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
# Build the generator
generator = build_generator()
# The generator takes noise as input and generates imgs
z = Input(shape=(100+384,))
img = generator(z)
# For the combined model we will only train the generator
discriminator.trainable = False
temp=Input(shape=(1,128,3))
# The discriminator takes generated images as input and determines validity
valid = discriminator([img,temp])
# The combined model (stacked generator and discriminator)
# Trains the generator to fool the discriminator
combined = Model(z, valid)
combined.compile(loss='binary_crossentropy', optimizer=optimizer)
The discriminator have 2 models, and will get as input an image of shape 128x128x3 and an embedding of shape 1x128x3 and both models are merged then. The generator model just gets noise and generates a 128x128x3 image. So at the line combined = Model(z, valid) I am getting the followiing error:
RuntimeError: Graph disconnected: cannot obtain value for tensor Tensor("input_5:0", shape=(?, 1, 128, 3), dtype=float32) at layer "input_5". The following previous layers were accessed without issue: ['input_4', 'model_2']
which I think is because of the fact that discriminator can't find embedding input but I am feeding it a tensor of shape (1,128,3), just like noise is being fed to the generator model. Can anyone please help me where I am doing wrong?
And after everything is set here is how I will generate images from noise and embedding vector merged together and discriminator will take image and vector to identify fakes:
#texts has embedding vectors
pics=np.array(pics) . #images
noise = np.random.normal(0, 1, (batch_size, 100))
j=0
latent_code=[]
for j in range(len(texts)): #appending embedding at the end of noise
n=np.append(noise[j],texts[j])
n=n.tolist()
latent_code.append(n)
latent_code=np.array(latent_code)
gen_imgs = generator.predict(latent_code) #gen making fakes
j=0
vects=[]
for im in gen_imgs:
t=np.array(texts[j])
t=np.reshape(t,[128,3])
t=np.expand_dims(t, axis=0)
vects.append(t)
j+=1
vects=np.array(vects) #vector of ?,1,128,3
#disc marking fakes and reals
d_loss_real = discriminator.train_on_batch([pics,vects], valid)
d_loss_fake = discriminator.train_on_batch([gen_pics,vects], fake)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
g_loss = combined.train_on_batch(latent_code, valid)
You have forgotten to add the temp as one of the inputs of the GAN (that's why the error says it can't feed the corresponding tensor since it is essentially disconnected):
combined = Model([z, temp], valid)
As a side note, I highly recommend to use Keras Functional API for building complicated and multi branch models like your discriminator. It is much easier to use, being more flexible and less error-prone.
For example, this is the descriminator you have written but I have rewritten it using Functional API. I personally think it is much easier to follow:
def build_discriminator(img_shape,embedding_shape):
input_img = Input(shape=img_shape)
x = Conv2D(32, kernel_size=5, strides=2, padding="same")(input_img)
x = LeakyReLU(alpha=0.2)(x)
x = Dropout(0.25)(x)
x = Conv2D(48, kernel_size=5, strides=2, padding="same")(x)
x = BatchNormalization(momentum=0.8)(x)
x = LeakyReLU(alpha=0.2)(x)
x = Dropout(0.25)(x)
x = Conv2D(64, kernel_size=5, strides=2, padding="same")(x)
x = BatchNormalization(momentum=0.8)(x)
x = LeakyReLU(alpha=0.2)(x)
x = Dropout(0.25)(x)
x = Conv2D(128, kernel_size=5, strides=2, padding="same")(x)
x = BatchNormalization(momentum=0.8)(x)
x = LeakyReLU(alpha=0.2)(x)
x = Dropout(0.25)(x)
x = Conv2D(256, kernel_size=5, strides=2, padding="same")(x)
x = BatchNormalization(momentum=0.8)(x)
x = LeakyReLU(alpha=0.2)(x)
x = Dropout(0.25)(x)
x = Flatten()(x)
output_img = Dense(200)(x)
input_emb = Input(shape=embedding_shape)
y = Dense(50)(input_emb)
y = Dense(100)(y)
y = Dense(200)(y)
output_emb = Flatten()(y)
merged = concatenate([output_img, output_emb])
output_merge = Dense(1, activation='sigmoid', name='output_layer')(merged)
return Model([input_img, input_emb], output_merge)