We Keep Coding

Visualizing the attention map of a multihead attention in ViT

I'm trying to visualize the attention map of mit Visual Transformer architecture in keras/tensorflow. For this I was able to implement the ViT model the following way:
def model():
input_layer = layers.Input(shape=input_shape)
#image_patches = create_patches(input_layer)
#print(input_layer.shape)
image_patches = Patches(patch_size)(input_layer)
#print(image_patches.shape)
encoded_patches = PatchEncoder(num_patch, projection_dim)(image_patches)
#print(encoded_patches.shape)
#for i in range(transformer_blocks):
x1 = layers.LayerNormalization()(encoded_patches)
x1 = layers.MultiHeadAttention(num_heads=num_heads, key_dim=projection_dim, name='MHA_1')(x1, x1)
x = layers.Add()([x1, encoded_patches])
x2 = layers.LayerNormalization()(x)
x2 = mlp_head(x2, transformer_units)
encoded_patches = layers.Add()([x2, x])
x = layers.LayerNormalization()(encoded_patches)
x = layers.Flatten()(x)
x = layers.Dropout(0.5)(x)
x = layers.Dense(2)(x)
model = tf.keras.Model(inputs=input_layer, outputs=x)
print(model.summary())
return model
I'm now trying to visualize the attention map based on an input image and my model output. For this I first try to predict the outcome and reshape the weights:
def attention_map(model, image):
size = model.input_shape[1]
grid_size = int(np.sqrt(model.layers[4].output_shape[-2] - 1))
# Prepare the input
X = preprocess_inputs(cv2.resize(image, (size, size)))#[np.newaxis, :] # type: ignore
# Get the attention weights from each transformer.
outputs = [
l.output[1] for l in model.layers if isinstance(l, layers.MultiHeadAttention)
]
weights = np.array(
tf.keras.models.Model(inputs=model.inputs, outputs=outputs).predict(X_test)
)
print(weights.shape)
num_layers = weights.shape[0]
num_heads = weights.shape[1]
reshaped = weights.reshape(
(num_layers, num_heads, grid_size ** 2 + 1, grid_size ** 2 + 1)
)
# From Appendix D.6 in the paper ...
# Average the attention weights across all heads.
reshaped = reshaped.mean(axis=1)
# From Section 3 in https://arxiv.org/pdf/2005.00928.pdf ...
# To account for residual connections, we add an identity matrix to the
# attention matrix and re-normalize the weights.
reshaped = reshaped + np.eye(reshaped.shape[1])
reshaped = reshaped / reshaped.sum(axis=(1, 2))[:, np.newaxis, np.newaxis]
# Recursively multiply the weight matrices
v = reshaped[-1]
for n in range(1, len(reshaped)):
v = np.matmul(v, reshaped[-1 - n])
# Attention from the output token to the input space.
mask = v[0, 1:].reshape(grid_size, grid_size)
mask = cv2.resize(mask / mask.max(), (image.shape[1], image.shape[0]))[
..., np.newaxis
]
return (mask * image).astype("uint8")
However my problem is now to reshape my weight matrix getting in mismatch. Can someone give me a hint on why this is occuring? A hint based on the output dimension given by
weights = np.array(
tf.keras.models.Model(inputs=model.inputs, outputs=outputs).predict(X_test)
)
would also help.

Using a multiple matrices to predict a matrix in Python with Keras

I modified the code from here. What I'm trying to do is combine the two matrices to predict the output matrix. The output matrix is built from the two input matrices. The problem seems to be associated to:
self.Combined_dense_1 = tf.keras.layers.Dense(units=32, activation="relu")
self.Combined_dense_2 = tf.keras.layers.Dense(units=16, activation="softmax")
The linked medium tutorial only predicting a single number based on the combined mixed input. I however am trying to predict a whole matrix but don't know how to structure the combined layer (if this is even the problem).
The error: "ValueError: Shape mismatch: The shape of labels (received (40,)) should equal the shape of logits except for the last dimension (received (10, 16))."
The code:
import warnings
import sys
if not sys.warnoptions:
warnings.simplefilter("ignore")
import numpy as np
import os
import random
import tensorflow as tf
from tensorflow import keras
from IPython.display import clear_output
class model(keras.Model):
def __init__(self):
super().__init__()
# The layers to process our image
self.Conv2D_1 = tf.keras.layers.Conv2D(filters=32,
kernel_size=(1, 1),
strides=(1, 1)
)
self.Conv2D_2 = tf.keras.layers.Conv2D(filters=32,
kernel_size=(3, 3),
strides=(1, 1)
)
# our combined layers
self.Combined_dense_1 = tf.keras.layers.Dense(units=32, activation="relu")
self.Combined_dense_2 = tf.keras.layers.Dense(units=16, activation="softmax")
def call(self, input_image_one, input_image_two):
# Image model
I = self.Conv2D_1(input_image_one)
I = self.Conv2D_2(I)
# Flatten I so we can merge our data.
I = tf.keras.layers.Flatten()(I)
N = self.Conv2D_1(input_image_two)
N = self.Conv2D_2(N)
N = tf.keras.layers.Flatten()(N)
# Combined model
x = tf.concat([N, I], 1) # Concatenate through axis #1
x = self.Combined_dense_1(x)
x = self.Combined_dense_2(x)
return x
network = model()
optimizer = tf.keras.optimizers.Adam()
loss_function = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
def train_step(model, optimizer, loss_function,
images_one_batch, images_two_batch,
labels):
with tf.GradientTape() as tape:
model_output = model(images_one_batch, images_two_batch)
print(model_output)
loss = loss_function(labels, model_output) # our labels vs our predictions
grads = tape.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(zip(grads, model.trainable_variables))
return loss
def train(model, optimizer, loss_function, epochs,
images_one_batch, images_two_batch,
labels):
loss_array = []
for epoch in range(epochs):
loss = train_step(model, optimizer, loss_function, images_one_batch, images_two_batch, labels)
loss_array.append(loss)
if ((epoch + 1) % 20 == 0):
# Calculating accuracy
network_output = network(images_one_batch, images_two_batch)
preds = np.argmax(network_output, axis=1)
acc = 0
for i in range(len(images_one_batch)):
if (preds[i] == labels[i]):
acc += 1
print(" loss:", loss, " Accuracy: ", acc / len(images_one_batch) * 100, "%")
clear_output(wait=True)
NumberofVars = 2;
width= NumberofVars; height = NumberofVars
NumberOfComputationSets = 10
CM_MatrixArr1 = []
CM_MatrixArr2 = []
for j in range(NumberOfComputationSets):
Theta1 = list(np.reshape(np.random.randint(2, size=4), (1,4))[0])
Theta1 = list(np.float_(Theta1))
CM_MatrixArr1.append(Theta1)
Theta2 = list(np.reshape(np.random.randint(2, size=4), (1,4))[0])
Theta2 = list(np.float_(Theta2))
CM_MatrixArr2.append(Theta2)
combinedCM_MatrixArr = []
combinedCM_toIntArr = []
for x,y in zip(CM_MatrixArr1, CM_MatrixArr2):
combinedCM = []
combinedCM_toInt = 0
for a,b in zip(x,y):
LogVal = (a == b)
combinedCM.append(float(LogVal == True))
combinedCM_MatrixArr.append(combinedCM)
combinedCM_MatrixArr = np.array(combinedCM_MatrixArr)
combinedCM_MatrixArr = combinedCM_MatrixArr.reshape(NumberOfComputationSets,2,2)
CM_MatrixArr1 = np.array(CM_MatrixArr1)
CM_MatrixArr1 = CM_MatrixArr1.reshape(NumberOfComputationSets,2,2)
CM_MatrixArr1 = CM_MatrixArr1.reshape(NumberOfComputationSets, 2,2,1)
CM_MatrixArr2 = np.array(CM_MatrixArr2)
CM_MatrixArr2 = CM_MatrixArr2.reshape(NumberOfComputationSets,2,2)
CM_MatrixArr2 = CM_MatrixArr2.reshape(NumberOfComputationSets, 2,2,1)
train(network,optimizer,loss_function,300,CM_MatrixArr1,CM_MatrixArr2,combinedCM_MatrixArr)

Transforming keras model output during training and use multiple losses

if self.use_embedding:
input_x = Input(shape=(self.input_length,), name='embedding_input')
x = Embedding(self.input_dim, self.embedding_dim)(input_x)
else:
input_x = Input(shape=(self.input_length,self.input_dim), name='notes_input')
x = input_x
encoder_input_list = [input_x]
encoded = self._build_encoder(x)
self.encoder = Model(inputs=encoder_input_list, outputs=encoded)
encoded_input = Input(shape=(self.latent_rep_size,), name='encoded_input')
if self.use_embedding:
input_decoder_x = Input(shape=(self.output_dim,), name='embedding_input_decoder_start')
#decoder_x = Embedding(self.output_dim, self.output_dim, input_length=1)(input_decoder_x)
decoder_x = input_decoder_x
else:
input_decoder_x = Input(shape=(self.output_dim,), name='input_decoder_start')
decoder_x = input_decoder_x
autoencoder_decoder_input_list = [input_decoder_x, encoded]
decoder_input_list = [input_decoder_x, encoded_input]
autoencoder_input_list = [input_x, input_decoder_x]
autoencoder_output_list = []
if self.teacher_force:
ground_truth_input = Input(shape=(self.output_length, self.output_dim), name='ground_truth_input')
decoder_input_list.append(ground_truth_input)
autoencoder_decoder_input_list.append(ground_truth_input)
autoencoder_input_list.append(ground_truth_input)
else:
ground_truth_input = None
if self.history:
history_input = Input(shape=(self.latent_rep_size,), name='history_input')
decoder_input_list.append(history_input)
autoencoder_decoder_input_list.append(history_input)
autoencoder_input_list.append(history_input)
else:
history_input = None
decoded= self._build_decoder(decoder_x, encoded_input, ground_truth_input, history_input)
loss_list = []
loss_weights_list = []
sample_weight_modes = []
loss_weights_list.append(1.0)
sample_weight_modes.append('temporal')
loss_list.append(self.vae_loss)
metrics_list = ['accuracy']
decoder_output = decoded
self.decoder = Model(inputs=decoder_input_list, outputs=decoder_output, name='decoder')
decoder_final_output = self.decoder(autoencoder_decoder_input_list)
if isinstance(decoder_final_output, list):
autoencoder_output_list.extend(decoder_final_output)
else:
autoencoder_output_list.append(decoder_final_output)
self.autoencoder = Model(inputs=autoencoder_input_list, outputs=autoencoder_output_list, name='autoencoder')
self.autoencoder.compile(optimizer=self.optimizer,
loss=loss_list,
loss_weights=loss_weights_list,
sample_weight_mode=sample_weight_modes,
metrics=metrics_list)
Here is my code for a variational sequence to sequence model. I want to transform the decoder output during the training so that It gives me two other matrices and use two different loss functions for the new matrices. So the total loss should be like L_total = L_mat1 + L_mat2 + L_mat3 - KL loss. Is there a way to do it in Keras? I have the ground truth values for the transformed matrices. Any good soul would help me with this?

I've added an example on how to implement a matrix split operation to this colab notebook:
https://colab.research.google.com/drive/127DY9OUWasFQzM9G2AH4RQO8ryhSTJny?usp=sharing
The short answer is that you can use a lambda to split the output of any layer.
In my simple example:
inputs = layers.Input(shape=(2,))
d = layers.Dense(10)(inputs)
out1, out2 = layers.Lambda(lambda x: (x[:, :5], x[:, 5:]))(d)
model = keras.Model(inputs, [out1, out2])
model.compile(loss=['mse', 'mae'])
return model
The lambda layer splits the first 5 columns of the matrix into out1 and the last 5 cols into out2.

Siamese network assigns same label to every couple of images

I'm working on a siamese network to identify if the input images are the same picture or not.
The problem is that the network keep assigning the label 0 (rather than 0 and 1) resulting in 0.5 accuracy every time but I don't know why.
Here's the model:
def Model():
input_dim = (200, 200, 1)
img_a = Input(shape = input_dim)
img_b = Input(shape = input_dim)
base_net = build_base_network(input_dim)
features_a = base_net(img_a)
features_b = base_net(img_b)
distance = Lambda(euclidean_distance, output_shape = eucl_dist_output_shape)([features_a, features_b])
model = Model(inputs=[img_a, img_b], outputs=distance)
return model
And here's the distance
def contrastive_loss(y_true, y_pred):
margin = 1
return K.mean(y_true * K.square(y_pred) + (1 - y_true) * K.square(K.maximum(margin - y_pred, 0)))
#Optimizer
rms = RMSprop()
#Distance
def euclidean_distance(vects):
x, y = vects
return K.sqrt(K.sum(K.square(x - y), axis=1, keepdims=True))
def eucl_dist_output_shape(shapes):
shape1, shape2 = shapes
return (shape1[0], 1)
Training and X/y shapes
#Pair_equal --> every element is a tuple of numpy arrays representing same images
#Pair_diff --> every element is a tuple of numpy arrays representing different images
#y_equal --> for every Pair equal's element, contains 0
#y_diff --> for every Pair equal's element, contains 1
if len(Pair_equal) > len(Pair_diff):
Pair_equal = Pair_equal[0:len(Pair_diff)]
y_equal = y_equal[0:len(y_diff)]
elif len(Pair_equal) < len(Pair_diff):
Pair_diff = Pair_diff[0:len(Pair_equal)]
y_diff = y_diff[0:len(y_equal)]
Pair_equal = np.array(Pair_equal)
Pair_diff = np.array(Pair_diff)
y_equal = np.array(y_equal)
y_diff = np.array(y_diff)
X = np.concatenate([Pair_equal, Pair_diff], axis=0)
y = np.concatenate([y_equal, y_diff], axis=0)
y = y.reshape(-1, 1)
#index shuffling
indices = np.arange(X.shape[0])
np.random.shuffle(indices)
X = X[indices]
y = y[indices]
#X shape: (32, 2, 200, 200, 1)
#y shape: (32, 1)
return [X[:,0,...], X[:,1,...]], y
Thank you in advance for the help.

RBF-Neural net can't classify MNIST dataset

I have implemented a RBF neural network classifier.
I use my implementation to classify the MNIST dataset, but it is not learning and always just predicts a single class. I would be very grateful if someone could help me identify the problem with my implementation.
I have to note that the implementation is quite slow due to the fact it works example by example, but I don't know how to make it such that it works batch by batch. (I am new to tensorflow and python in general)
My implementation is as follows:
class RBF_NN:
def __init__(self, M, K, L, lr):
#Layer sizes
self.M = M #input layer size - number of features
self.K = K #RBF layer size
self.L = L #output layer size - number of classes
#
x = tf.placeholder(tf.float32,shape=[M])
matrix = tf.reshape(tf.tile(x,multiples=[K]),shape=[K,M])
prototypes_input = tf.placeholder(tf.float32,shape=[K,M])
prototypes = tf.Variable(prototypes_input) # prototypes - representatives of the data
r = tf.reduce_sum(tf.square(prototypes-matrix),1)
s = tf.Variable(tf.random.uniform(shape=[K],maxval=1)) #scaling factors
h = tf.exp(-r/(2*tf.pow(s,2)))
W = tf.Variable(tf.random.uniform(shape=[K,L],maxval=1))
b = tf.Variable(tf.constant(0.1, shape=[L]))
o = tf.matmul(tf.transpose(tf.expand_dims(h,1)),W) + b
pred_class = tf.argmax(o,1)
y = tf.placeholder(shape=[L], dtype=tf.float32)
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=o, labels=y))
optimizer = tf.train.AdamOptimizer(lr).minimize(loss)
self.x = x
self.prototypes_input = prototypes_input
self.prototypes = prototypes
self.r = r
self.s = s
self.h = h
self.W = W
self.b = b
self.o = o
self.y = y
self.loss = loss
self.optimizer = optimizer
self.pred_class = pred_class
def fit(self,X,y,prototypes,epoch_count,print_step,sess):
for epoch in range(epoch_count):
epoch_loss = 0
for xi,yi in zip(X,y):
iter_loss, _ = sess.run((self.loss,self.optimizer),feed_dict={self.x: xi, self.y: yi, self.prototypes_input:prototypes})
epoch_loss = epoch_loss + iter_loss
epoch_loss = epoch_loss/len(X)
if epoch%print_step == 0:
print("Epoch loss",(epoch+1),":",epoch_loss)
def predict(self,x,sess):
return sess.run((self.pred_class),feed_dict={self.x:x})[0]
def get_prototypes(self,sess):
return sess.run((self.prototypes))
Usage:
mnist = tf.keras.datasets.mnist
(x_train, y_train),(x_test, y_test) = mnist.load_data()
y_train = to_one_hot(y_train,10)
y_test = to_one_hot(y_test,10)
x_train = np.asarray([np.asarray(x).reshape(-1) for x in x_train])
x_test = np.asarray([np.asarray(x).reshape(-1) for x in x_test])
M = 784
K = 1000
L = 10
lr = 0.01
rbfnn = RBF_NN(M,K,L,lr)
#Selecting prototypes from the train set
idx = np.random.randint(len(x_train), size=K)
prototypes = x_train[idx,:]
init = tf.global_variables_initializer()
sess = tf.InteractiveSession()
sess.run(init,feed_dict={rbfnn.prototypes_input:prototypes})
rbfnn.fit(x_train,y_train,prototypes,epoch_count=1, print_step=1,sess=sess)
y_test_p = []
for xi,yi in zip(x_test,y_test):
yp = rbfnn.predict(xi,sess=sess)
y_test_p.append(yp)
y_test_t = [np.argmax(yi) for yi in y_test]
acc = accuracy_score(y_test_t,y_test_p,)
precc = precision_score(y_test_t,y_test_p, average='macro')
recall = recall_score(y_test_t,y_test_p, average = 'macro')
f1 = f1_score(y_test_t,y_test_p,average='macro')
print("Accuracy:",acc)
print("Precision:",precc)
print("Recall:",recall)
print("F1 score:",f1)
sess.close()

The implementation is fine. However, it seems to be very sensitive to the data.
It will start learning just fine if the following lines are added:
x_train = (x_train-x_train.min())/(x_train.max()-x_train.min())
x_test = (x_test-x_test.min())/(x_test.max()-x_test.min())
In this way the data is normalized so that the interval of each feature is from 0 to 1.