We Keep Coding

ValueError Input shape in a custom layer tensorflow

I have this custom layer:
class PhysicalLayer(keras.layers.Layer):
def __init__(self,units, speed):
self.units = units
self.speed = speed
super(PhysicalLayer, self).__init__()
def build(self, input_shape):
self.w = self.add_weight(shape=(input_shape[-1], self.units), initializer="random_normal", trainable=True)
self.b = self.add_weight(shape=(self.units,), initializer="random_normal", trainable=True)
def call(self, inputs):
squareSpeed = tf.math.square(self.speed)
vibrationMax = tf.math.reduce_max(inputs, axis = 1, keepdims = True)
inputsSpeed = tf.math.divide(squareSpeed, vibrationMax)
print(tf.shape(inputsSpeed))
print(tf.shape(self.w))
multiplication = tf.multiply(squareSpeed, self.w)
return tf.matmul(inputsSpeed , self.w) + self.b
#return multiplication + self.b
And when I try to build this following model
inputs = keras.Input(shape=(500,))
dense = layers.Dense(64, activation="relu")
x = PhysicalLayer(1, rotationSpeed)(inputs)
x = dense(x)
x = layers.Dense(32, activation="relu")(x)
outputs = layers.Dense(1)(x)
modelPhi = keras.Model(inputs=inputs, outputs=outputs, name="model_phi_custom")
I have the following error:
ValueError: Dimensions must be equal, but are 1 and 500 for '{{node physical_layer_34/MatMul}} = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false](physical_layer_34/truediv, physical_layer_34/MatMul/ReadVariableOp)' with input shapes: [?,1], [500,1].
I tried to use multiplication instead of matmul but then for fitting the model, I need to use a batch_size of 500 or i come across this error:
Node: 'gradient_tape/mean_absolute_error/sub/BroadcastGradientArgs'Incompatible shapes: [500,1] vs. [64,1]
How can I fix this ?
Thanks

Keras subclassing style model training huge performance difference

I was running DSSM_N and DSSM on a dataset with batch size 512 on 2060.
However,
DSSM_N costs ~35ms per batch
DSSM. costs ~400ms per batch.
What makes this huge performance difference? I have checked profiling which said
that DSSM costs ~350ms on All Others Time. How can I fix the DSSM implementation?
Many thanks in advance.
Edited as suggested by Micheal:
The main difference is DSSM makes a hash-table-like lookup (notice tf.nn.embedding_lookup and IntegerLookup) which makes the dataset preprocess a little bit simpler while in DSSM_N this lookup was done in dataset preprocess in advance. However, I don't believe this simple hash table like makes such a big difference. What was I doing wrong?
import pickle
import tensorflow as tf
import tensorflow.keras as keras
import tensorflow_hub as hub
import tensorflow_text as text # required for BERT hub model
from keras.layers import Layer, Embedding, Dense, Concatenate, BatchNormalization, Dropout, Dot, Hashing, TextVectorization, GRU, IntegerLookup
from keras import Model
import random
from ..config import *
from ..util import *
def embedding_sequence_reduce_mean(x, mask):
# float[B,L,E], bool[B,L] -> float[B,E]
x = tf.ragged.boolean_mask(x, mask) # (B, Lr, E) remove masked data
x = tf.reduce_mean(x, axis=1) # (B, E)
x = tf.where(tf.math.is_nan(x), 0.0, x) # nan to 0
return x
def embedding_masked_to_zero(x, mask):
mask = tf.expand_dims( # B -> B 1 align for broadcasting
tf.cast(mask, dtype=tf.float32), axis=1)
return x * mask
USER_ID_DIM = 128
MEDIA_ID_DIM = 64
GENRE_DIM = 32
ORIGIN_DIM = 32
LATENT_DIM = latent_dim
N_HASH = 8
N_BIN = 1024
print('N_HASH', N_HASH)
print('N_BIN', N_BIN)
class HashEmbedding(Layer):
# TODO: with_importance is not supported
def __init__(
self, n_hash, n_bin, output_dim,
embeddings_initializer='uniform', embeddings_regularizer=None,
activity_regularizer=None, embeddings_constraint=None,
mask_zero=False, input_length=None, **kwargs
):
super(HashEmbedding, self).__init__()
self.mask_zero = mask_zero
self.n_hash = n_hash
self.n_bin = n_bin
# salts no duplication
self.salts = random.sample(range(self.n_hash * 32), self.n_hash)
self.hashs = [Hashing(
num_bins=self.n_bin,
# if mask_zero then hash 0 to 0
mask_value=(0 if self.mask_zero else None),
salt=self.salts[i])
for i in range(self.n_hash)]
self.embedding = Embedding(
self.n_bin, output_dim,
embeddings_initializer=embeddings_initializer,
embeddings_regularizer=embeddings_regularizer,
activity_regularizer=activity_regularizer,
embeddings_constraint=embeddings_constraint,
mask_zero=mask_zero, input_length=input_length
)
def compute_mask(self, inputs, mask=None):
if not self.mask_zero:
return None
return tf.not_equal(inputs, 0)
def call(self, inputs):
shape = inputs.shape
hash = tf.stack([hash(inputs) # [I], n_hash
for hash in self.hashs], axis=len(shape))
x = self.embedding(hash) # [I], n_hash, emb_dim
x = tf.reduce_sum(x, axis=len(shape)) # [I], emb_dim
return x
class StringVectorization(Layer):
def __init__(self, vocab, embedding_dim=32, output_dim=16):
super(StringVectorization, self).__init__()
self.text_vectorization = TextVectorization(
vocabulary=vocab, split='character')
self.embedding = Embedding(
self.text_vectorization.vocabulary_size(), embedding_dim, mask_zero=True)
self.gru = GRU(output_dim)
def call(self, inputs): # B, S
x = self.text_vectorization(inputs)
x = self.embedding(x)
return self.gru(x)
class TfBertZh(Layer): # 128 - 2 input length limit
def __init__(self): # output_dim 768
super(TfBertZh, self).__init__()
self.preprocess = hub.KerasLayer(
zh_preprocessor_model_file, trainable=False)
self.encoder = hub.KerasLayer(zh_encoder_model_file, trainable=False)
def call(self, inputs):
x = self.preprocess(inputs)
x = self.encoder(x)['pooled_output']
return x
class DNN(Layer):
def __init__(self):
super(DNN, self).__init__()
self.concat = Concatenate(axis=1)
self.dense1 = Dense(64)
self.bn = BatchNormalization()
self.drop = Dropout(0.1)
self.dense2 = Dense(32)
def call(self, inputs: list):
from keras.activations import tanh
x = self.concat(inputs)
x = self.drop(tanh(self.bn(self.dense1(x))))
x = tanh(self.dense2(x))
return x
with open(stats_file_pkl, 'rb') as f:
sinfo = pickle.load(f)
with open(vocab_file_pkl, 'rb') as f:
vocab = pickle.load(f)
class DSSM_N(Model):
def __init__(self):
super(DSSM_N, self).__init__()
self.user_id = HashEmbedding(
N_HASH, N_BIN, USER_ID_DIM, mask_zero=True)
self.item_id = Embedding(
sinfo['media_id']['unique'], MEDIA_ID_DIM, mask_zero=True)
self.genre = Embedding(
sinfo['genre_id']['unique'], GENRE_DIM, mask_zero=True)
self.origin = Embedding(
sinfo['origin_id']['unique'], ORIGIN_DIM, mask_zero=True)
self.user_dnn = DNN()
self.item_dnn = DNN()
self.dot = Dot(axes=1, normalize=False)
def call(self, inputs):
u = self.compute_user_latent({'id': inputs['user']})
n_pos = inputs['pos'].shape[1]
n_neg = inputs['neg'].shape[1]
ui_pos = []
ui_neg = []
def signal(u, i):
return tf.exp(self.dot([u, i]))
for j in range(n_pos):
i = self.compute_item_latent({
'id': inputs['pos'][:, j],
'genre': inputs['pos_genre'][:, j, :], # B N 4
'origin': inputs['pos_origin'][:, j, :] # B N 2
})
ui_pos.append(signal(u, i))
ui_pos = tf.add_n(ui_pos)
for j in range(n_neg):
i = self.compute_item_latent({
'id': inputs['neg'][:, j],
'genre': inputs['neg_genre'][:, j, :],
'origin': inputs['neg_origin'][:, j, :]
})
ui_neg.append(signal(u, i))
ui_neg = tf.add_n(ui_neg)
return tf.squeeze(ui_pos / (ui_pos + ui_neg))
def compute_user_latent(self, inputs):
id = self.user_id(inputs['id'])
latent = self.user_dnn([id])
return latent
def compute_item_latent(self, inputs):
id = self.item_id(inputs['id'])
genre = self.genre(inputs['genre']) # B 4 -> B 4 E
genre = embedding_sequence_reduce_mean(genre, genre._keras_mask)
origin = self.origin(inputs['origin']) # B 2 -> B 2 E
origin = embedding_sequence_reduce_mean(origin, origin._keras_mask)
latent = self.item_dnn([id, genre, origin])
return latent
user_df = pd.read_pickle(preprocessed_user_file_pkl)
media_df = pd.read_pickle(preprocessed_media_file_pkl)
genre_df = pd.read_pickle(clean_genre_file_pkl)
origin_df = pd.read_pickle(clean_origin_file_pkl)
class MediaPreprocess(Layer):
def __init__(self):
super(MediaPreprocess, self).__init__()
self.lookup = IntegerLookup(vocabulary=list(media_df['id']))
self.genre_table = tf.Variable(
[[0] * 4] + list(media_df['genre']), dtype=tf.int32, trainable=False)
self.origin_table = tf.Variable(
[[0] * 2] + list(media_df['origin']), dtype=tf.int32, trainable=False)
self.id_embedding = Embedding(
self.lookup.vocabulary_size() + 1, MEDIA_ID_DIM, mask_zero=True)
self.genre_embedding =\
Embedding(genre_df['id'].max() + 1, GENRE_DIM, mask_zero=True)
self.origin_embedding =\
Embedding(origin_df['id'].max() + 1, ORIGIN_DIM, mask_zero=True)
def __call__(self, inputs):
index = self.lookup(inputs) # B -> B
vector = self.id_embedding(index) # B -> B E
vector = embedding_masked_to_zero(vector, vector._keras_mask)
genre = tf.nn.embedding_lookup(self.genre_table, index)
genre = self.genre_embedding(genre)
genre = embedding_sequence_reduce_mean(genre, genre._keras_mask)
origin = tf.nn.embedding_lookup(self.origin_table, index)
origin = self.origin_embedding(origin)
origin = embedding_sequence_reduce_mean(origin, origin._keras_mask)
return {
'id': vector,
'genre': genre,
'origin': origin}
class UserPreprocess(Layer):
def __init__(self):
super(UserPreprocess, self).__init__()
self.lookup = IntegerLookup(vocabulary=list(user_df['id']))
self.embedding = HashEmbedding(
N_HASH, N_BIN, USER_ID_DIM, mask_zero=True)
def __call__(self, inputs):
vector = self.embedding(inputs)
vector = embedding_masked_to_zero(vector, vector._keras_mask)
return {'id': vector}
class DSSM(Model):
def __init__(self, *args, **kwargs):
super(DSSM, self).__init__()
self.user_pp = UserPreprocess()
self.item_pp = MediaPreprocess()
self.user_nn = DNN()
self.item_nn = DNN()
dot = Dot(axes=1, normalize=False)
self.signal = lambda u, i: tf.exp(dot([u, i]))
def call(self, inputs):
user = inputs['user'] # B
pos_s = inputs['pos'] # B N_POS=1
neg_s = inputs['neg'] # B N_NEG=7
n_pos = pos_s.shape[1]
n_neg = neg_s.shape[1]
u = self.user_pp(user)['id'] # B E(uid)
u = self.user_nn([u]) # B L
def compute_ui(i_s, count):
ui = []
for j in range(count):
i = self.item_pp(i_s[:, j])
i = self.item_nn([i['id'], i['genre'], i['origin']])
ui.append(self.signal(u, i))
return tf.add_n(ui) # C B 1 -> B 1
pos_ui = compute_ui(pos_s, n_pos) # B 1
neg_ui = compute_ui(neg_s, n_neg) # B 1
return tf.squeeze(pos_ui / (neg_ui + pos_ui)) # B

Convolutional Neural Network does not reach an acceptable accuracy

Howdy!
Recently I have built my own library for neural networks.
Without convolutional layers it worked fine. However, now, that I have implemented convolutional layers, it doesn't improve (in comparison to the dense nn) at all, which is inacceptable for more complex tasks like for instance Pneumonia Detection.
For backprop, every layer updates it's own values and passes it's input gradient to the layer behind it. For forwardprop every layer just gives its output as input to the next layer.
In the example below the model is set up for pneumonia detection with 128px * 128px images.
The accuracy allways stays under 60% no matter how much it's trained.
Here is the relevant code:
model = nG.NeuralNetwork([
nG.Convolutional(1, (128, 128), 3, 8),
nG.Pooling(2),
nG.ReLU(),
nG.Convolutional(8, (63, 63), 3, 16),
nG.Pooling(2),
nG.ReLU(),
nG.Convolutional(16, (31, 31), 3, 16),
nG.Pooling(2),
nG.ReLU(),
nG.Convolutional(16, (15, 15), 3, 32),
nG.Pooling(2),
nG.ReLU(),
nG.Convolutional(32, (7, 7), 3, 64),
nG.ReLU(),
nG.Reshape((64, 5, 5), (1600, 1)),
nG.Dense(1600, 128),
nG.Tanh(),
nG.Dense(128, 2),
nG.Tanh()],
nG.MSE()
)
model.train(images, labels, epochs=3, lr=0.01)
class NeuralNetwork:
def __init__(self, layers, loss):
self.layers = layers
self.loss = loss
def forwardProp(self, input):
output = input
for layer in self.layers:
output = layer.forwardProp(output)
return output
def backwardProp(self, errorDeriv, lr):
deltaOutput = errorDeriv
for layer in reversed(self.layers):
deltaOutput = layer.backwardProp(deltaOutput, lr)
def train(self, xTrain, yTrain, epochs=1, lr=1, interimResult=False):
corrects = 0
print("Precompiling ... This might take a few seconds", end="\n\n")
for epoch in range(epochs):
print(f"{epoch+1}th epoch:")
round_start = time.time()
i = -1
for X, Y in zip(xTrain, yTrain):
i += 1
start = time.time()
output = self.forwardProp(X)
errorDeriv = self.loss.errorDerivative(output, Y)
self.backwardProp(errorDeriv, lr)
--
class Convolutional():
def __init__(self, input_depth, input_size, kernel_size, depth):
self.input_depth = input_depth
self.input_size = input_size
self.kernel_size = kernel_size
self.depth = depth
self.kernels = np.random.uniform(-0.5, 0.5, (depth, input_depth, kernel_size, kernel_size))
self.bias = [np.random.uniform(-0.5, 0.5, (input_size[0] - kernel_size + 1, input_size[1] - kernel_size + 1)) for i in range(depth)]
self.input = None
def forwardProp(self, input):
self.input = input
output = get_output(input, self.depth, self.input_size, self.kernel_size, self.input_depth, self.kernels, self.bias)
return output
def backwardProp(self, output_delta, lr):
kernels_gradient, input_delta = get_gradients(self.kernels, self.input, self.depth, self.input_depth, output_delta)
self.kernels -= lr * kernels_gradient
self.bias -= lr * output_delta
return input_delta
#numba.njit
def get_gradients(kernels, input, depth, input_depth, output_delta):
kernels_gradient = np.zeros(kernels.shape)
input_delta = np.zeros(input.shape)
for i in range(depth):
for j in range(input_depth):
kernels_gradient[i, j] = valid_correlate(input[j], output_delta[i])
input_delta[j] += full_convolve(output_delta[i], kernels[i, j])
return kernels_gradient, input_delta
#numba.njit(fastmath=True, nogil=True)
def get_output(input, depth, input_size, kernel_size, input_depth, kernels, bias):
out = np.zeros((depth, input_size[0] - kernel_size + 1, input_size[0] - kernel_size + 1))
for k in range(depth):
for i in range(input_depth):
out[k] += valid_correlate(input[i], kernels[k][i])
out[k] += bias[k]
return out
class Pooling:
def __init__(self, size):
self.size = size
self.input = None
def forwardProp(self, input):
self.input = input
output = []
for i in range(input.shape[0]):
output.append(pool(input[i], self.size))
output = np.asarray(output)
return output
def backwardProp(self, output_delta, lr):
input_delta = anti_pool(output_delta, self.input.shape, self.size, self.input)
return input_delta
def anti_pool(output_delta, input_shape, size, input):
input_delta = np.zeros(input_shape)
for l in range(input_delta.shape[0]):
for x in range(output_delta.shape[1]):
for y in range(output_delta.shape[2]):
area_start = (x * size, y * size)
area_end = (min((x + 1) * size, input_delta.shape[1]),
min((y + 1) * size, input_delta.shape[2]))
area = (input[l, area_start[0]:area_end[0], area_start[1]:area_end[1]])
highest_pos = np.unravel_index(area.argmax(), area.shape)
highest_pos = [x * size + highest_pos[0],
y * size + highest_pos[1]]
input_delta[l, highest_pos[0], highest_pos[1]] = output_delta[l, x, y]
return input_delta
#numba.njit("float64[:,:](float64[:,:], int32)")
def pool(mat, size):
def pool_at_position(mat, pos):
end_pos = (min(mat.shape[0], pos[0] + size),
min(mat.shape[1], pos[1] + size))
area = mat[pos[0]:end_pos[0], pos[1]:end_pos[1]]
result = np.max(area)
return result
output_size = (int(np.ceil(mat.shape[0] / size)), int(np.ceil(mat.shape[1] / size)))
output = np.zeros(output_size)
for x in range(output_size[0]):
for y in range(output_size[1]):
output[x, y] = pool_at_position(mat, (x * size, y * size))
return output
class Dense:
def __init__(self, inputSize, outputSize):
self.weights = np.random.randn(outputSize, inputSize)
self.bias = np.random.randn(outputSize, 1)
def forwardProp(self, input):
self.input = input
return np.dot(self.weights, self.input) + self.bias
def backwardProp(self, output_gradient, lr):
weights_gradient = np.dot(output_gradient, self.input.T)
input_gradient = np.dot(self.weights.T, output_gradient)
self.weights -= lr * weights_gradient
self.bias -= lr * output_gradient
return input_gradient
class Tanh:
def __init__(self):
self.input = None
self.output = None
def forwardProp(self, input):
self.input = input
self.output = tanh(input)
return self.output
def backwardProp(self, outputDelta, lr):
inputDelta = 1 - (np.tanh(self.input) ** 2)
inputDelta *= outputDelta
return inputDelta
#numba.vectorize
def tanh(x):
return np.tanh(x)
class ReLU:
def __init__(self):
self.input = None
self.output = None
def forwardProp(self, input):
self.input = input
self.output = np.maximum(input, 0)
return self.output
def backwardProp(self, outputDelta, lr):
inputDelta = np.multiply(outputDelta, np.vectorize(self.anti_relu)(self.input))
return inputDelta
def anti_relu(self, x):
if x < 0:
return 0
else:
return 1
class MSE :
def __init__(self):
pass
def errorFunction(self, output, Y):
error = (output - Y) ** 2
return error
def errorDerivative(self, output, Y):
error_deriv = 2 * (output - Y)
return error_deriv
For the functions/Classes that I've not included I'm dead sure that they work.
I spent the last couple of days reading over the code and still haven't found the problem yet.
I would be extremely thankful for any kind of response.
Kind Regards
Eirik

How to use torch.nn.transformer with pytroch lightning?

I am trying to use the vanilla transformer from PyTorch using Pytorch Lightning. I tried to test the model with a reverse number task. So given [1, 3, 5, 4, 13, 19] it returns [1, 13, 4, 5, 3, 19] with 1, 19 being start and end token respectively. The full code is below. The code can run without error but there seems to be a problem with the backpropagation. The training loss does go down at first but it doesn't go beyond 2.8 and the accuracy doesn't go beyond 11%.
It seems that part of the model is able to optimize, I am guessing it is because the weights located in Embeddings and Generator can backpropagate, but weights located in nn.Transformer cannot? I am really not sure.
import math
import torch.nn.functional as F
import numpy as np
import torch
import torch.nn as nn
from torch.utils.data import DataLoader
import pytorch_lightning as pl
from pytorch_lightning.callbacks import EarlyStopping
class Embeddings(pl.LightningModule):
def __init__(self, d_model, vocab):
super(Embeddings, self).__init__()
self.lut = nn.Embedding(vocab, d_model)
self.d_model = d_model
def forward(self, x):
a = self.lut(x) * math.sqrt(self.d_model)
return a
class PositionalEncoding(pl.LightningModule):
def __init__(self, d_model, dropout, max_len=5000):
super(PositionalEncoding, self).__init__()
self.dropout = nn.Dropout(p=dropout)
# Compute the positional encodings once in log space.
pe = torch.zeros(max_len, d_model)
position = torch.arange(0, max_len).unsqueeze(1)
div_term = torch.exp(torch.arange(0, d_model, 2) *
-(math.log(10000.0) / d_model))
pe[:, 0::2] = torch.sin(position * div_term)
pe[:, 1::2] = torch.cos(position * div_term)
pe = pe.unsqueeze(0)
self.register_buffer('pe', pe)
def forward(self, x):
x = x + self.pe[:, :x.size(1)]
return self.dropout(x)
class Generator(pl.LightningModule):
def __init__(self, size):
super(Generator, self).__init__()
self.proj = nn.Linear(512, size)
def forward(self, x):
return F.log_softmax(self.proj(x), dim=-1)
class Model(pl.LightningModule):
def __init__(self, src_embed, tgt_embed, transformer, generator):
super(Model, self).__init__()
self.src_embed = src_embed
self.tgt_embed = tgt_embed
self.transformer = transformer
self.generator = generator
self.valLoss = 0
self.valAcc = 0
self.automatic_optimization = False
self.optimizer = None
for p in self.parameters():
if p.dim() > 1:
nn.init.xavier_uniform_(p)
def forward(self, x, y, tgt_mask=None):
x = self.src_embed(x)
y = self.tgt_embed(y)
return self.generator(self.transformer(x, y, tgt_mask=tgt_mask))
def training_step(self, batch, batch_idx):
if self.optimizer is None:
self.optimizer = self.optimizers()
batch = Batch(batch[0], batch[1])
tgt_mask = batch.trg_mask.squeeze(0)
tgt_mask = (tgt_mask != True)
output = self(batch.src, batch.trg, tgt_mask)
criterion = LossCompute(V)
loss = criterion.forward(output.contiguous().view(-1, output.size(-1)), batch.trg_y.contiguous().view(-1)) / batch.ntokens
loss.backward()
self.optimizer.step()
self.optimizer.zero_grad()
self.log('train_loss', loss)
print(loss)
def validation_step(self, batch, batch_idx):
batch = Batch(batch[0], batch[1])
tgt_mask = batch.trg_mask.squeeze(0)
tgt_mask = (tgt_mask != True)
output = self(batch.src, batch.trg, tgt_mask)
criterion = LossCompute(V)
loss = criterion.forward(output.view(-1, output.size(-1)), batch.trg_y.contiguous().view(-1)) / batch.ntokens
self.log('val_loss', loss)
self.valLoss += loss
if batch_idx % 10 == 0:
print(loss)
if batch_idx == 99:
print(self.valLoss/100)
self.valLoss = 0
return {"x": output, "trg": batch.trg_y, "index": batch_idx}
def validation_step_end(self, batch):
output, trg, idx = batch["x"], batch["trg"], batch["index"]
accuracy = getAccuracy(output, trg)
self.log("accuracy", accuracy)
self.valAcc += accuracy
if idx == 99:
print(self.valAcc/100)
self.valAcc = 0
def train_dataloader(self):
data = data_gen(V, 0, 3000)
return DataLoader(data, batch_size=30, shuffle=False, num_workers=2, pin_memory=True)
def val_dataloader(self):
data = data_gen(V, 1, 1000)
return DataLoader(data, batch_size=10, shuffle=False, num_workers=2, pin_memory=True)
def configure_optimizers(self):
return torch.optim.Adam(self.parameters(), lr=1e-3, betas=(0.9, 0.98), eps=1e-9)
class LossCompute(pl.LightningModule):
def __init__(self, size):
super(LossCompute, self).__init__()
self.criterion = nn.KLDivLoss(reduction='sum')
self.size = size
self.true_dist = None
def forward(self, x, target):
# x has size (batch_size x length, vocab_size)
assert x.size(1) == self.size
true_dist = x.data.clone()
true_dist.fill_(0)
true_dist.scatter_(1, target.data.unsqueeze(1).long(), 1)
self.true_dist = true_dist
return self.criterion(x, true_dist)
# prepare data
class Batch:
"Object for holding a batch of data with mask during training."
def __init__(self, src, trg=None):
self.src = src
if trg is not None:
self.trg = trg[:, :-1]
self.trg_y = trg[:, 1:]
self.trg_mask = \
self.make_std_mask(self.trg)
self.ntokens = self.trg_y.size(0) * self.trg_y.size(1)
print("")
#staticmethod
def make_std_mask(tgt):
"Create a mask to hide padding and future words."
tgt_mask = subsequent_mask(tgt.size(-1)).type_as(tgt.data)
return tgt_mask
def subsequent_mask(size):
"Mask out subsequent positions."
attn_shape = (1, size, size)
subsequent_mask = np.triu(np.ones(attn_shape), k=1).astype('uint8')
return torch.from_numpy(subsequent_mask) == 0
def data_gen(V, randomSeed, totalTrainingSample):
np.random.seed(randomSeed)
x = torch.from_numpy(np.random.randint(2, V - 2, size=(totalTrainingSample, 10)))
y = torch.flip(torch.flip(x, [0, 1]), [0])
x[:, 0] = 1
y[:, 0] = 1
x[:, -1] = V - 1
y[:, -1] = V - 1
return list(zip(x, y))
def getAccuracy(x, trg):
totalValAcc = 0
totalValAccToken = 0
trg = trg.contiguous().view(-1)
out = x.view(-1, x.size(-1)) # (batch_size * tgt_length, src_vocab)
_, index = torch.max(out, dim=-1) # index (batch_size * tgt_length)
correct = list((trg == index)).count(True)
totalValAcc += correct
totalValAccToken += index.size(0)
return totalValAcc / totalValAccToken
V = 20
transformer = nn.Transformer(num_encoder_layers=2, num_decoder_layers=2, batch_first=True)
PositionEnc = PositionalEncoding(512, 0.1)
src_emb = Embeddings(512, V)
tgt_emb = Embeddings(512, V)
gen = Generator(V)
if __name__ == '__main__':
model = Model(nn.Sequential(src_emb, PositionEnc), nn.Sequential(tgt_emb, PositionEnc), transformer, gen)
earlyStopping = EarlyStopping(monitor='val_loss', patience=3)
trainer = pl.Trainer(max_epochs=10, callbacks=[earlyStopping])
trainer.fit(model)

How to do fully connected batch norm in PyTorch?

torch.nn has classes BatchNorm1d, BatchNorm2d, BatchNorm3d, but it doesn't have a fully connected BatchNorm class? What is the standard way of doing normal Batch Norm in PyTorch?

Ok. I figured it out. BatchNorm1d can also handle Rank-2 tensors, thus it is possible to use BatchNorm1d for the normal fully-connected case.
So for example:
import torch.nn as nn
class Policy(nn.Module):
def __init__(self, num_inputs, action_space, hidden_size1=256, hidden_size2=128):
super(Policy, self).__init__()
self.action_space = action_space
num_outputs = action_space
self.linear1 = nn.Linear(num_inputs, hidden_size1)
self.linear2 = nn.Linear(hidden_size1, hidden_size2)
self.linear3 = nn.Linear(hidden_size2, num_outputs)
self.bn1 = nn.BatchNorm1d(hidden_size1)
self.bn2 = nn.BatchNorm1d(hidden_size2)
def forward(self, inputs):
x = inputs
x = self.bn1(F.relu(self.linear1(x)))
x = self.bn2(F.relu(self.linear2(x)))
out = self.linear3(x)
return out

The BatchNorm1d normally comes before the ReLU, and the bias is redundant, so
import torch.nn as nn
class Policy(nn.Module):
def __init__(self, num_inputs, action_space, hidden_size1=256, hidden_size2=128):
super(Policy2, self).__init__()
self.action_space = action_space
num_outputs = action_space
self.linear1 = nn.Linear(num_inputs, hidden_size1, bias=False)
self.linear2 = nn.Linear(hidden_size1, hidden_size2, bias=False)
self.linear3 = nn.Linear(hidden_size2, num_outputs)
self.bn1 = nn.BatchNorm1d(hidden_size1)
self.bn2 = nn.BatchNorm1d(hidden_size2)
def forward(self, inputs):
x = inputs
x = F.relu(self.bn1(self.linear1(x)))
x = F.relu(self.bn2(self.linear2(x)))
out = self.linear3(x)
return out