We Keep Coding

How to plot Receptive Fields, for a CNN/fashionMNIST?

I created my CNN with PyTorch Lightning, and I am actually looking for plotting the Receptive Fields.
Do you have any suggestions about it?
I look for different solutions here and there, but I actually can't make them synergize with PyTorch Lightning.
Is it possible to visualize the Receptive fields directly inside Tensorboard?
I'll share with you my Dataset:
train_dataset = torchvision.datasets.FashionMNIST('classifier_data', train=True, download=True, transform=transforms.ToTensor())
train, val = train_test_split(train_dataset, test_size = .2)
train_loader = DataLoader(train, batch_size = 32)
val_loader = DataLoader(train, batch_size = 32)
test_dataset = torchvision.datasets.FashionMNIST('classifier_data', train=False, download=True, transform=transforms.ToTensor())
test_loader = DataLoader(test_dataset, batch_size = 32)
and CNN:
def __init__(self, dropout, learn_rate, momentum, weight_decay, optimizer):
#def __init__(self, dropout, learn_rate, weight_decay, optimizer):
super().__init__()
self.conv1 = nn.Conv2d(in_channels = 1, out_channels = 6, kernel_size = 5)
self.conv2 = nn.Conv2d(in_channels = 6, out_channels = 12 , kernel_size = 5)
self.fc1 = nn.Linear(in_features = 12*4*4, out_features = 120)
self.fc2 = nn.Linear(in_features = 120, out_features = 60)
self.out = nn.Linear(in_features = 60, out_features = 10)
self.do = nn.Dropout(dropout) #for overfitting issues
self.loss = nn.CrossEntropyLoss()
self.accuracy = torchmetrics.Accuracy()
self.learn_rate = learn_rate
self.momentum = momentum #with Adam we don't have momentum. To Check best Optimizer with Optune, please comment this line.
self.weight_decay = weight_decay
self.optimizer = optimizer
self.train_loss = []
self.val_loss = []
self.train_acc = []
self.test_acc = []
#plot into tensorboard
log_dir = pathlib.Path.cwd() / "lightning_logs"
self.writer = SummaryWriter(log_dir)
#forward step
#I add each layer to the histogram. It's plotted into tensorboard
def forward(self, x, additional_out=False):
#conv1
x = self.conv1(x)
self.writer.add_histogram("First convolutional layer CNN", x)
x = F.relu(x)
x = F.max_pool2d(x, kernel_size = 2, stride = 2)
#conv2
x = self.conv2(x)
self.writer.add_histogram("Second convolutional layer CNN", x)
x = F.relu(x)
x = F.max_pool2d(x, kernel_size = 2, stride = 2)
#fuly connected 1
x = x.reshape(-1, 12*4*4)
x = self.fc1(x)
self.writer.add_histogram("First linear layer CNN", x)
x = F.relu(x)
x = self.do(x)
#fully connected 2
x=self.fc2(x)
self.writer.add_histogram("Second linear layer CNN", x)
x = F.relu(x)
x = self.do(x)
#output
x = self.out(x)
self.writer.add_histogram("Output layer CNN", x)
return x
#optimizer
def configure_optimizers(self):
#optimizer = self.optimizer(self.parameters(), lr = self.learn_rate, momentum = self.momentum, weight_decay = self.weight_decay)
optimizer = self.optimizer(self.parameters(), lr = self.learn_rate, weight_decay = self.weight_decay)
return optimizer
#training step
def training_step(self, batch, batch_idx):
x, y = batch
b = x.size(0)
x = x.view(b, -1, 28, 28)
logit = self(x)
J = self.loss(logit, y) #loss
#self.train_loss.append(J) #no need to append
acc = self.accuracy(logit, y) #accuracy
#self.train_acc.append(acc) #no need to append
self.log("train_loss_cnn", J.item())
self.log("train_acc_cnn", acc.item())
return {'loss': J}
#Since I used Tensorboard, it don't have to append to loss
def test_step(self, batch, batch_idx):
p, q = batch
b = p.size(0)
p = p.view(b, -1, 28, 28)
logit = self(p)
J = self.loss(logit, q) #loss
acc_test = self.accuracy(logit, q) #accuracy
#self.train_acc.append(acc_test) #no need to append
#self.train_loss.append(J) #no need to append
self.log("test_acc_cnn", acc_test.item())
self.log("test_loss_cnn", J.item())
def validation_step(self, batch, batch_idx=None):
u, v = batch
b = u.size(0)
u = u.view(b, -1, 28, 28)
logit = self(u)
J = self.loss(logit, v) #loss
#self.val_loss.append(J) #no need to append
acc_val = self.accuracy(logit, v) #accuracy
#self.train_acc.append(acc_val) #no need to append
self.log("val_loss_cnn", J.item())
self.log("val_acc_cnn", acc_val.item())
return {"loss": J, "pred": logit, "target": v}
#Once saves from validation step, I take with me the returned elements, and I can plot the Confusion Matrix inside Tensorboard
def validation_epoch_end(self, outputs):
preds = torch.cat([tmp['pred'] for tmp in outputs])
targets = torch.cat([tmp['target'] for tmp in outputs])
conf_mat = confusion_matrix(preds, targets, num_classes=10)
df_cm = pd.DataFrame(conf_mat.numpy(), index = range(10), columns=range(10))
plt.figure(figsize = (10,7))
fig_ = sns.heatmap(df_cm, annot=True, cmap='Spectral').get_figure()
plt.close(fig_)
self.logger.experiment.add_figure("Confusion matrix CNN", fig_, self.current_epoch)

Why the train loss and train acc never change about the GNCNN (Gaussian-Neuron CNN)?

This is my first time to train a model myself, and I have some serious problem.
I know this problem has already been discussed here.
https://discuss.pytorch.org/t/why-the-train-loss-and-train-acc-never-change/101334
But I still can't fix it.
My code has slightly changed a little compared to the original one, but the problems are the same as description below.
Problem:
The code can run but the train loss and train acc never change
train_loss = 0.69, train_acc = 0.5.
I think the model does not be trained, but I can’t find my fault.
I try all solution I can find, change lr, reset_parameters, normalize and so on.
Maybe the preprocessing? The image are Gray image
COVER = 1
STEGO = 0
class CustomImageDataset(Dataset):
# [
# (path_0, label_0),
# ...
# (path_n, label_n)
# ] tuple
def __init__(self, img_dir): #training_dataset & testing_dataset
self.data = []
self.readData(img_dir + '/cover_png', COVER)
self.readData(img_dir + '/stego_png', STEGO)
print(self.data)
def readData(self, path, label):
file_names = os.listdir(path)
for file_name in file_names:
file_path = path + '/' + file_name
self.data.append((file_path, label))
def __len__(self):
return len(self.data)
def __getitem__(self, idx):
img_path, label = self.data[idx]
image = read_image(img_path)
return image, label
def gaussian1(x):
mean = torch.mean(x)
std = torch.std(x)
return torch.exp(-((x-mean)**2)/(torch.std(x))**2)
def gaussian2(x):
mean = torch.mean(x)
std = torch.std(x)
return 0.5 * torch.exp(-((x-mean)**2)/(torch.std(x))**2)
KV = torch.tensor([[-1,2,-2,2,-1],
[2,-6,8,-6,2],
[-2,8,-12,8,-2],
[2,-6,8,-6,2],
[-1,2,-2,2,-1]])/12.
KV = KV.view(1,1,5,5).to(device=device, dtype=torch.float)
KV = torch.autograd.Variable(KV, requires_grad=False)
class GNCNN(nn.Module):
def __init__(self):
super(GNCNN, self).__init__()
self.gaussian1 = gaussian1
self.gaussian2 = gaussian2
self.conv1 = nn.Conv2d(1, 16, kernel_size=5, stride=1, padding=0, bias=True)
self.avg_pool1 = nn.AvgPool2d(kernel_size=3, stride=2, padding=1)
self.conv2 = nn.Conv2d(16, 16, kernel_size=3, stride=1, padding=0, bias=True)
self.avg_pool2 =nn.AvgPool2d(kernel_size=3, stride=2, padding=1)
self.conv3 = nn.Conv2d(16, 16, kernel_size=3, stride=1, padding=0, bias=True)
self.avg_pool3 = nn.AvgPool2d(kernel_size=3, stride=2, padding=0)
self.conv4 = nn.Conv2d(16, 16, kernel_size=3, stride=1, padding=0, bias=True)
self.avg_pool4 = nn.AvgPool2d(kernel_size=3, stride=2, padding=0)
self.conv5 = nn.Conv2d(16, 16, kernel_size=5, stride=1, padding=0, bias=True)
self.avg_pool5 = nn.AvgPool2d(kernel_size=3, stride=2, padding=0)
self.fc1 = nn.Linear(16*4*4, 128)
self.fc2 = nn.Linear(128, 128)
self.fc3 = nn.Linear(128, 2)
self.reset_parameters()
def forward(self, x):
prep = F.conv2d(x, KV, padding=2) # preprocessing
out = self.avg_pool1(gaussian1(self.conv1(prep)))
out = self.avg_pool2(gaussian2(self.conv2(out)))
out = self.avg_pool3(gaussian2(self.conv3(out)))
out = self.avg_pool4(gaussian2(self.conv4(out)))
out = self.avg_pool5(gaussian2(self.conv5(out)))
out = out.view(out.size(0), -1)
out = F.relu(self.fc1(out))
out = F.relu(self.fc2(out))
out = self.fc3(out)
return out
def reset_parameters(self):
for mod in self.modules():
if isinstance(mod, nn.Conv2d):
nn.init.xavier_uniform_(self.conv1.weight)
elif isinstance(mod, nn.Linear):
nn.init.kaiming_normal_(mod.weight.data)
def accuracy(outputs, labels):
_, argmax = torch.max(outputs, 1)
return (labels == argmax.squeeze()).float().mean()
def default_loader(path):
try:
img = Image.open(path)
return img.convert('RGB')
except:
print("Cannot read image: {}".format(path))
def train_model(model, criterion, optimizer, num_epochs, batch_size, use_gpu):
since = time.time()
best_model_wts = model.state_dict()
best_acc = 0.0
val_acc_history = []
val_loss_history = []
is_best = False
for epoch in range(num_epochs):
begin_time = time.time()
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
count_batch = 0
epoch_loss = 0.
epoch_accuracy = 0.
running_loss = 0.
running_accuracy = 0.
val_loss = 0.
val_accuracy = 0.
test_loss = 0.
test_accuracy = 0.
for inputs, labels in train_dataloader:
# print(inputs[0].shape)
# for data in train_loader:
# # count_batch += 1
# inputs, labels = data
inputs = inputs.float()
if use_gpu:
inputs = Variable(inputs.cuda())
labels = Variable(labels.cuda())
else:
inputs, labels = Variable(inputs), Variable(labels)
optimizer.zero_grad()
outputs = model(inputs)
###
accuracys = accuracy(outputs, labels).item()
print("accuracy = ", accuracys)
running_accuracy += accuracys
epoch_accuracy += accuracys
###
loss = criterion(outputs, labels)
running_loss += loss.item()
epoch_loss += loss.item()
loss.backward()
optimizer.step()
print("epch loss = ", epoch_loss)
epoch_loss /= (train_sizes/batch_size)
train_loss_history.append(epoch_loss)
print("epch acc = ", epoch_accuracy)
#epoch_accuracy /= train_sizes
epoch_accuracy /= (train_sizes / batch_size)
train_acc_history.append(epoch_accuracy)
print('\nTrain: Epoch [{}] Loss: {:.4f} Acc: {:.4f}%'.format(epoch, epoch_loss, 100*epoch_accuracy))
print('\nTrain: Epoch [{}] Loss: {:.4f}'.format(epoch, epoch_loss))
if __name__ == '__main__':
use_gpu = torch.cuda.is_available()
batch_size = 10
learning_rate = 0.001
# atransforms = transforms.Compose([
# transforms.Resize((256, 256)),
# transforms.Grayscale(1),
# # transforms.RandomResizedCrop(224),
# transforms.RandomHorizontalFlip(),
# transforms.ToTensor(),
# transforms.Normalize([0.485, ], [0.229, ])
# ])
training_dataset = CustomImageDataset('/content/gdrive/MyDrive/Topic_Research/toy_BOSSBase/train')
testing_dataset = CustomImageDataset('/content/gdrive/MyDrive/Topic_Research/toy_BOSSBase/test')
train_sizes = len(training_dataset)
print(train_sizes)
train_dataloader = DataLoader(training_dataset, batch_size, shuffle=True)
test_dataloader = DataLoader(testing_dataset, batch_size, shuffle=True)
model = GNCNN()
################
#model.reset_parameters()
if use_gpu:
model = model.cuda()
criterion = nn.CrossEntropyLoss().cuda()
optimizer = optim.SGD(model.parameters(), lr = learning_rate, momentum=0.9)
train_acc_history = []
train_loss_history = []
model = train_model(model=model, criterion=criterion, optimizer=optimizer,
num_epochs=10, batch_size=batch_size, use_gpu=use_gpu)
We are using the BOSSbase_1.01 dataset,HUGO algorithm 0.4bpp and 8000
train set, 2000 val set, epoch = 50 but always get train_acc = 50% and
train_loss = 69% our result

Accuracy, training, test loss doesn't change throughout all the epochs

I am working on a text classification problem with a binary output 0 or 1. The accuracy, train loss and test loss remains the same. The accuracy is exact the same for all the epochs. All the steps looks very correct. I had tried out several ways to figure out what is going wrong. Nothing actually worked. Please help.
class RNN(nn.Module):
def __init__(self, num_layers, num_classes, input_size, hidden_size,vocab,dropout):
super(RNN,self).__init__()
self.num_layers = num_layers
self.num_classes = num_classes
self.input_size = input_size
self.hidden_size = hidden_size
self.sequence_len = sequence_len
self.embedding = nn.Embedding(len(vocab),input_size)
nn.init.xavier_normal_(self.embedding.weight)
self.rnn = nn.RNN(input_size, hidden_size,num_layers,dropout = dropout, nonlinearity = 'tanh', batch_first=True, bias = True, bidirectional = False)
self.linear = nn.Linear(hidden_size, 1)
nn.init.xavier_normal_(self.linear.weight)
self.sigmoid = nn.Sigmoid()
def forward(self,x):
lens = list(map(len, x))
padded = pad_sequence(x, batch_first=True)
output_embedding=self.embedding(padded)
packed = pack_padded_sequence(input = output_embedding,lengths = lens, batch_first=True, enforce_sorted=False)
input_rnn = packed
h011 = torch.zeros(1,32,3)
output_11, hidden_11 = self.rnn(input_rnn,h011)
output_padded, output_lengths = pad_packed_sequence(output_11, batch_first=True)
final_output_11 = self.linear(hidden_11)
prob_11 = self.sigmoid(final_output_11)
return output_padded, hidden_11, prob_11
num_layers = 1
num_classes = 2
input_size = 5
hidden_size = 3
criterion = nn.BCELoss()
sequence_len = 1
dropout = 0.5
rnn = RNN(num_layers, num_classes, input_size, hidden_size,vocab,dropout)
epochs = 10
lr = 0.01
weight_decay=0.011
def train_loop(model,criterion,optimizer,train_loader,valid_loader,epochs):
train_losses= []
valid_losses= []
for epoch in range(epochs):
train_loss=0
for label,text in train_loader:
output,hidden,prob = rnn.forward(text)
prob = torch.tensor([item.item() for sublist in prob for item in sublist],dtype = torch.float32)
label = torch.tensor(label, dtype = torch.float32)
loss=criterion(prob,label)
optimizer.zero_grad()
loss.requires_grad = True
loss.backward()
optimizer.step()
train_loss += loss.item()
train_loss=train_loss/len(train_loader)
valid_loss=0
with torch.no_grad():
correct=0
total=0
for label,text in valid_loader:
output,hidden,prob = rnn.forward(text)
prob = torch.tensor([item.item() for sublist in prob for item in sublist],dtype = torch.float32)
label = torch.tensor(label, dtype = torch.float32)
loss=criterion(prob,label)
valid_loss += loss.item()
p = torch.tensor([1 if i > 0.5 else 0 for i in prob.data], dtype = torch.float32)
#predicted = torch.max(p, 1)
total += label.size(0)
correct += (p == label).sum().item()
valid_loss=valid_loss/len(valid_loader)
accuracy = 100 * correct / total
print(accuracy)
scheduler.step(accuracy)
train_losses.append(train_loss)
valid_losses.append(valid_loss)
print(f'Epoch {epoch+1:<2d}/{epochs} --> Train Loss: {train_loss:.4f} | Valid Loss: {valid_loss:.4f}')
from torch.optim.lr_scheduler import StepLR
from torch.optim.lr_scheduler import ReduceLROnPlateau
optimizer = torch.optim.Adam(rnn.parameters(), lr=0.01)
scheduler = ReduceLROnPlateau(optimizer, mode='max', factor=0.4, patience=5,
verbose=True) # need to change scheduler.step
train_loop(rnn,criterion,optimizer,train_loader,valid_loader,epochs)
Output:

RuntimeError: expected scalar type Long but found Int in loss = criterion(outputs, y_train)

I built this acoustic model with features dim = [1124823,13] and labels dim = [1124823,1] and I split both to train, test, and dev. The problem that when I try to run my model I get this error
RuntimeError: expected scalar type Long but found Int in
loss = criterion(outputs, y_train)
import torch
import torch.nn as nn
from fela import feat, labels
from Dataloader import train_loader, test_loader, X_train, X_test, X_val, y_train, y_test, y_val
################################################################################################
input_size = 13
hidden1_size = 13
hidden2_size = 128
hidden3_size = 64
output_size = 50
################################################################################################
class DNN(nn.Module):
def __init__(self, input_size, hidden2_size, hidden3_size, output_size):
super(DNN, self).__init__()
self.fc1 = nn.Linear(input_size, hidden1_size)
self.relu1 = nn.ReLU()
self.fc2 = nn.Linear(hidden1_size, hidden2_size)
self.relu2 = nn.ReLU()
self.fc3 = nn.Linear(hidden2_size, hidden3_size)
self.relu3 = nn.ReLU()
self.fc4 = nn.Linear(hidden3_size, output_size)
self.relu4 = nn.ReLU()
def forward(self, x):
out = self.fc1(x)
out = self.relu1(out)
out = self.fc2(out)
out = self.relu2(out)
out = self.fc3(out)
out = self.relu3(out)
out = self.fc4(out)
out = self.relu4(out)
return out
################################################################################################
# Instantiate the model
batch_size = 50
n_iterations = 50
no_epochs = 80
model = DNN(input_size, hidden2_size, hidden3_size, output_size)
################################################################################################
# Define the loss criterion and optimizer
criterion = nn.CrossEntropyLoss()
learning_rate = 0.01
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)
print(model)
########################################################################################################################
# train the network
iter = 0
for epoch in range(no_epochs):
for i, (X_train, y_train) in enumerate(train_loader):
optimizer.zero_grad()
outputs = model(X_train)
loss = criterion(outputs, torch.max(labels, 1)[1])
loss.backward()
optimizer.step()
iter += 1
if iter % 500 == 0:
correct = 0
total = 0
for X_test, y_test in test_loader:
outputs = model(X_test)
_, predicted = torch.max(outputs.data, 1)
total += labels.size(0)
correct += (predicted == labels).sum()
accuracy = 100 * correct / total
print(iter, loss.data[0], accuracy)

I think no_epochs=0 with this initialization. Possibly (len(train_loader) / batch_size) > n_iterations. Then int(no_eps) = 0. Try to change no_epochs to 100 manually, for example.
no_eps = n_iterations / (len(train_loader) / batch_size)
no_epochs = int(no_eps)
for epoch in range(no_epochs):

My Pytorch CNN computes but does not show training results

I am writing a CNN to classify some images. when I run the training part of my code, the only output I receive is the print "Training finished". This tells me that my code was iterated through but did not successfully compute - I have no errors and have really tried to identify the error.
The fc1's (First connected layer) input size is supposed to be the flattened previous layer.
Data
train_split = 0.70 # Defines the ratio of train/valid/test data.
valid_split = 0.10
train_size = int(len(data_df)*train_split)
valid_size = int(len(data_df)*valid_split)
ins_dataset_train = ImageNet10(
df=data_df[:train_size],
transform=data_transform,
)
ins_dataset_valid = ImageNet10(
df=data_df[train_size:(train_size +
valid_size)].reset_index(drop=True),
transform=data_transform,
)
ins_dataset_test = ImageNet10(
df=data_df[(train_size + valid_size):].reset_index(drop=True),
transform=data_transform,
)
Data Loaders
train_loader = torch.utils.data.DataLoader(
ins_dataset_train,
batch_size=16,
shuffle=True,
num_workers=2
)
valid_loader = torch.utils.data.DataLoader(
ins_dataset_valid,
batch_size=16,
shuffle=True,
num_workers=2
)
test_loader = torch.utils.data.DataLoader(
ins_dataset_test,
batch_size=24, # Forward pass only so batch size can be larger
shuffle=False,
num_workers=2
)
Convolutional neural network
class ConvNet(nn.Module):
def __init__(self, num_classes=10):
super(ConvNet, self).__init__()
# Add network layers here
# Layer 1
self.conv1 = nn.Conv2d(3,16, (3,3))
self.pool = nn.MaxPool2d(2,2)
self.relu = nn.ReLU()
self.dropout = nn.Dropout(p=0.3)
# Layer 2
self.conv2 = nn.Conv2d(16,24, (4,4))
self.pool = nn.MaxPool2d(2,2)
self.relu = nn.ReLU()
self.dropout = nn.Dropout(p=0.3)
# Layer 3
self.conv3 = nn.Conv2d(24,32, (4,4))
self.pool = nn.MaxPool2d(2,2)
self.relu = nn.ReLU()
self.dropout = nn.Dropout(p=0.3)
# Layer 4 - Fully connected
self.fc1 = nn.Linear(32 * 8 * 8, 512)
self.fc2 = nn.Linear(512, num_classes)
self.final = nn.Softmax(dim=1)
def forward(self, x):
out = x.reshape(x.size(0), -1) # TODO what does this do? Why do we need it?
out = self.fc1(out)
return out
Training Model
def train_model_epochs(num_epochs):
for epoch in range(10):
running_loss = 0.0
for i, data in enumerate(test_loader, 0):
images, labels = data
print("Batch", i, "size:", len(images))
optimizer.zero_grad()
outputs = net(images)
loss = loss_function(outputs, labels)
loss.backward()
optimizer.step()
running_loss =+ loss.item()
if i%1000 == 999:
print('Epoch / Batch [%d / %d] - Loss: %.3f' %
(epoch + 1, i + 1, running_loss / 1000))
running_loss = 0.0
print("Training finished")