I made the following simple toy CNN Model.
class Test(nn.Module):
def __init__(self):
super(Test, self).__init__()
self.conv = nn.Sequential(
nn.Conv2d(3,300,3),
nn.Conv2d(300,500,3),
nn.Conv2d(500,1000,3),
)
self.fc = nn.Linear(168200000,1)
def forward(self, x):
out = self.conv(x)
out = out.view(-1)
out = self.fc(out)
return out
and I made following trainer script.
if __name__ == '__main__':
device = 'cuda'
#Once
model = Test()
model.to(device)
optimizer = optim.Adam(model.parameters(), lr=0.001)
criterion = nn.MSELoss()
#Twice
model = Test()
model.to(device)
optimizer = optim.Adam(model.parameters(), lr=0.001)
for i in range(10):
optimizer.zero_grad()
print('i :', i)
x = torch.zeros((50,3,64,64), device=device)
y = torch.ones(50, device=device)
output = model(x)
loss = criterion(output, y)
loss.backward()
optimizer.step()
I have 12211MB of VRAM.
If I load model just once, I use 11878MB.
But if I load model twice, I use 9302MB.
I can save the memory 2576MB! I just loaded the model twice.
Why happen this?
Related
I'm trying to do machine learning using mnist dataset in pytorch, but the loss function doesn't decrease under 1.5.
I want the loss function to decrease under 1.
What to do for this problem?
what code should I fix?
This is my code:
BATCH_SIZE = 8
transform =torchvision.transforms.Compose([torchvision.transforms.ToTensor(),torchvision.transforms.Normalize((0.5,), (0.5,))])
trainset = torchvision.datasets.MNIST(root = "./data/train", train = True, download = True, transform = transform)
trainset, valset = torch.utils.data.random_split(trainset, [50000, 10000])
trainloader = torch.utils.data.DataLoader(trainset, batch_size = BATCH_SIZE, shuffle = True)
valloader = torch.utils.data.DataLoader(valset, batch_size = BATCH_SIZE, shuffle = True)
testset = torchvision.datasets.MNIST(root = "./data/test", train = False, download = True, transform = transform)
testloader = torch.utils.data.DataLoader(testset, batch_size = BATCH_SIZE, shuffle = False
class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
self.relu = nn.ReLU()
self.pool = nn.MaxPool2d(2, stride=2)
self.conv1 = nn.Conv2d(1,20,5)
self.conv2 = nn.Conv2d(20,50,5)
self.conv3 = nn.Conv2d(50,500,4)
self.conv4 = nn.Conv2d(500,10,1)
self.soft = nn.Softmax(dim=1)
def forward(self, x):
x = self.conv1(x)
x = self.pool(x)
x = self.conv2(x)
x = self.pool(x)
x = self.conv3(x)
x = self.relu(x)
x = self.conv4(x)
x = self.soft(x)
x = x.view(x.size()[0], -1)
return x
class EarlyStopping:
"""earlystopping class"""
def __init__(self, patience=5, verbose=False, path='checkpoint_model.pth'):
self.patience = patience
self.verbose = verbose
self.counter = 0
self.best_score = None
self.early_stop = False
self.val_loss_min = np.Inf
self.path = path
def __call__(self, valid_loss):
score = -valid_loss
if self.best_score is None:
self.best_score = score
self.checkpoint(valid_loss)
elif score - self.best_score < 0.0001:
self.counter += 1
if self.verbose:
print(f'EarlyStopping counter: {self.counter} out of {self.patience}')
if self.counter >= self.patience:
self.early_stop = True
else:
self.best_score = score
self.checkpoint(valid_loss)
self.counter = 0
def checkpoint(self, valid_loss):
if self.verbose:
print(f'Validation loss decreased ({self.val_loss_min:.6f} --> {valid_loss:.6f}).')
self.val_loss_min = valid_loss
earlystopping = EarlyStopping(patience=5, verbose=True)
net = Net()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=0.00005, momentum=0.9, weight_decay=0.005)
train_loss=[]
train_acc=[]
val_loss=[]
val_acc=[]
if __name__ == '__main__':
BATCH_SIZE = 8
start = time.time()
for epoch in range(10000):
print('epoch', epoch+1)
sum_loss = 0.0
sum_correct = 0
sum_total = 0
#Training
net = net.train()
for (inputs, labels) in trainloader:
#inputs, labels = inputs.to(device), labels.to(device)
optimizer.zero_grad()
outputs = net(inputs)
loss = criterion(outputs, labels)
sum_loss += loss.item() #Add loss
_, predicted = outputs.max(1)
sum_total += labels.size(0)
sum_correct += (predicted == labels).sum().item()
loss.backward()
optimizer.step()
print("train mean loss={}, accuracy={}"
.format(sum_loss*BATCH_SIZE/len(trainloader.dataset), float(sum_correct/sum_total)))
train_loss.append(sum_loss*BATCH_SIZE/len(trainloader.dataset))
train_acc.append(float(sum_correct/sum_total))
sum_loss = 0.0
sum_correct = 0
sum_total = 0
#Validating
net = net.eval()
for (inputs, labels) in valloader:
#inputs, labels = inputs.to(device), labels.to(device)
optimizer.zero_grad()
outputs = net(inputs)
loss = criterion(outputs, labels)
sum_loss += loss.item()
_, predicted = outputs.max(1)
sum_total += labels.size(0)
sum_correct += (predicted == labels).sum().item()
loss.backward()
optimizer.step()
print("valid mean loss={}, accuracy={}"
.format(sum_loss*BATCH_SIZE/len(valloader.dataset), float(sum_correct/sum_total)))
val_loss.append(sum_loss*BATCH_SIZE/len(valloader.dataset))
val_acc.append(float(sum_correct/sum_total))
#Early stop
earlystopping(val_loss[-1])
if earlystopping.early_stop:
print("Early Stopping!")
break
finish_time = time.time() - start
#Test
net.test()
test_loss = 0
correct = 0
with torch.no_grad():
for (inputs, labels) in testloader:
output = net(inputs)
test_loss += criterion(output, labels).item()
pred = output.argmax(dim=1, keepdim=True)
correct += pred.eq(labels.view_as(pred)).sum().item()
test_loss /= 10000
test_acc = correct / 10000
And this is the resulting image.
https://i.stack.imgur.com/eln4S.png
The loss function converges without going below 1.5, and earlystopping terminates machine learning.
Since I have done a similar task (with a different but comparable network) I know that the Loss when using cross_entropy decreases lower than 1.5. In my case accuracy was ~95% and the loss ~0.200. When inspecting your picture I saw that for the test loss (in the picture you named it Loss evaluate) the value matches with what one should expect for train/val loss after some epochs. So there is most likely an error when computing the loss for the training and validation phases.
You compute the loss for a certain epoch like this:
train_loss.append(sum_loss*BATCH_SIZE/len(trainloader.dataset))
len(trainloader.dataset) does not return number of batches but rather number of samples, thus using BATCH_SIZE for computing the loss is correct and necessary. See here.
Since Cross_Entropy by default reports the mean for the batch (see here) using
sum_loss*BATCH_SIZE/len(trainloader.dataset)
is correct if the trainloader actually has a Batch-size equal to the one defined in the provided code. Since you did not share the part where the trainloader was defined I cannot be sure.
What indicates this behaviour is that in the test part you don't use len(testloader.dataset) but rather hardcoded 10000 to get the mean for the epoch, I think that the dataloader (for all train, validate and testing) may not have batch size equal to BATCH_SIZE and is most likely 1.
Thanks for your time~ after I run the code, the grad is always zero and the loss is not updating.(I guess it's because the weights is initialized all 0's, but I don't know how to fix it) The code is a basic neural network:
class Model(torch.nn.Module): #class
def __init__(self):
super(Model, self).__init__()
self.linear1 = torch.nn.Linear(8,6)
self.linear2 = torch.nn.Linear(6,4)
self.linear3 = torch.nn.Linear(4,1)
self.sigmoid = torch.nn.Sigmoid()
def forward(self, x):
x = self.sigmoid(self.linear1(x))
x = self.sigmoid(self.linear2(x))
x = self.sigmoid(self.linear3(x))
x = F.softmax(x, dim=1)
return x
model = Model() #model
criterion = torch.nn.BCELoss(size_average = False)
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
for epoch in range(1000): # training
y_pred = model(X_train.float())
loss = criterion(y_pred, y_train.float())
print(epoch, loss.item())
print([x.grad for x in optimizer.param_groups[0]['params']])
optimizer.zero_grad()
loss.backward()
optimizer.step()
And I get the 0s grad:
I think your forgot to apply the backpropagation. Adding loss.backward() just before your print statetements will do the trick (compute the accumulated gradients and store them in x.grad). Note that by default your weights are not initialized to 0 here. The default initialization for linear layers are here.
I am working on a problem statement related to python classes:
I have two classes:
class MCC(object):
def __init__(self, problem_type, batch_size, dataset):
self.problem_type = problem_type
self.batch_size = batch_size
self.dataset = dataset
self.cls_weights = weights_calculation()
def weights_calculation(self):
class_weights = (1 - (self.dataset['labels'].value_counts().sort_index()/len(self.dataset))).values
return class_weights
second class
from transformers import Trainer
class WeightedTrainer(Trainer):
def compute_loss(self, model, inputs, return_outputs=False):
outputs = model(**inputs)
logits = outputs.get('logits')
labels = inputs.get('labels')
loss_func = nn.CrossEntropyLoss(weight = self.class_weights)
loss = loss_func(logits, labels)
return (loss, outputs) if return_outputs else loss
In the second class, I have to pass weight in nn.CrossEntropyLoss as in code loss_func = nn.CrossEntropyLoss(weight = self.class_weights)
I want to modify the inherited Trainer class to pass a new parameter custom_class_weight in MCC class.
What I have tried:
class MCC(object):
def __init__(self, problem_type, batch_size, dataset, model):
self.problem_type = problem_type
self.batch_size = batch_size
self.dataset = dataset
self.model = model
self.cls_weights = weights_calculation()
self.WeightedTrainer = WeightedTrainer(self.cls_weights)
self.trainer = self.WeightedTrainer(self.model)
def weights_calculation(self):
class_weights = (1 - (self.dataset['labels'].value_counts().sort_index()/len(self.dataset))).values
return class_weights
class WeightedTrainer(Trainer):
def __init__(self, custom_class_weight):
self.custom_class_weight = custom_class_weight
def compute_loss(self, model, inputs, return_outputs=False):
outputs = model(**inputs)
logits = outputs.get('logits')
labels = inputs.get('labels')
loss_func = nn.CrossEntropyLoss(weight = self.custom_class_weight)
loss = loss_func(logits, labels)
return (loss, outputs) if return_outputs else loss
Which is disturbing the inherited's Trainer class properties and thus giving me an error:
TypeError: 'WeightedTrainer' object is not callable
I also tried:
class WeightedTrainer(Trainer):
def __init__(self, custom_class_weight, **kwargs,):
self.custom_class_weight = custom_class_weight
super(WeightedTrainer, self).__init__(**kwargs)
def compute_loss(self, model, inputs, return_outputs=False):
outputs = model(**inputs)
logits = outputs.get('logits')
labels = inputs.get('labels')
loss_func = nn.CrossEntropyLoss(weight = self.custom_class_weight)
loss = loss_func(logits, labels)
return (loss, outputs) if return_outputs else loss
How to create a child class, using the Trainer class as a parent class and modify an inherited class while keeping all default properties of super class?
Thank you for #JMA valuable comment, Here is the fix.
I was calling self.WeightedTrainer which is not a callable object. Full working code:
class MCC(object):
def __init__(self, problem_type, batch_size, dataset, model):
self.problem_type = problem_type
self.batch_size = batch_size
self.dataset = dataset
self.model = model
self.cls_weights = weights_calculation()
self.custom_trainer = WeightedTrainer
self.trainer = self.WeightedTrainer(self.model, custom_class_weight = self.cls_weights)
def weights_calculation(self):
class_weights = (1 - (self.dataset['labels'].value_counts().sort_index()/len(self.dataset))).values
return class_weights
class WeightedTrainer(Trainer):
def __init__(self, custom_class_weight, **kwargs,):
super().__init__(**kwargs)
self.custom_class_weight = custom_class_weight
def compute_loss(self, model, inputs, return_outputs=False):
outputs = model(**inputs)
logits = outputs.get('logits')
labels = inputs.get('labels')
loss_func = nn.CrossEntropyLoss(weight = self.custom_class_weight)
loss = loss_func(logits, labels)
return (loss, outputs) if return_outputs else loss
I tried to put a droprate to my neural network (NN) using torch and I got a strange error at the end. How can I fix it?
So the idea is that I wrote a NN inside a function to make it easier to call. The function is the following:
(I personally think the problem lies inside the class of the NN, but for the sake of having a working example I'm putting everything).
def train_neural_network(data_train_X, data_train_Y, batch_size, learning_rate, graph = True, dropout = 0.0 ):
input_size = len(data_test_X.columns)
hidden_size = 200
num_classes = 4
num_epochs = 120
batch_size = batch_size
learning_rate = learning_rate
# The class of NN
class NeuralNet(nn.Module):
def __init__(self, input_size, hidden_size, num_classes, p = dropout):
super(NeuralNet, self).__init__()
self.fc1 = nn.Linear(input_size, hidden_size)
self.fc2 = nn.Linear(hidden_size, hidden_size)
self.fc3 = nn.Linear(hidden_size, num_classes)
def forward(self, x, p = dropout):
out = F.relu(self.fc1(x))
out = F.relu(self.fc2(out))
out = nn.Dropout(out, p) #drop
out = self.fc3(out)
return out
# Prepare data
X_train = torch.from_numpy(data_train_X.values).float()
Y_train = torch.from_numpy(data_train_Y.values).float()
# Loading data
train = torch.utils.data.TensorDataset(X_train, Y_train)
train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size)
net = NeuralNet(input_size, hidden_size, num_classes)
# Loss
criterion = nn.CrossEntropyLoss()
# Optimiser
optimiser = torch.optim.SGD(net.parameters(), lr=learning_rate)
# Proper training
total_step = len(train_loader)
loss_values = []
for epoch in range(num_epochs+1):
net.train()
train_loss = 0.0
for i, (predictors, results) in enumerate(train_loader, 0):
# Forward pass
outputs = net(predictors)
results = results.long()
results = results.squeeze_()
loss = criterion(outputs, results)
# Backward and optimise
optimiser.zero_grad()
loss.backward()
optimiser.step()
# Update loss
train_loss += loss.item()
loss_values.append(train_loss / batch_size )
print('Finished Training')
return net
And when I call the function:
net = train_neural_network(data_train_X = data_train_X, data_train_Y = data_train_Y, batch_size = batch_size, learning_rate = learning_rate, dropout = 0.1)
The error is the following:
net = train_neural_network(data_train_X = data_train_X, data_train_Y = data_train_Y, batch_size = batch_size, learning_rate = learning_rate, dropout = 0.1)
/usr/local/lib/python3.6/dist-packages/torch/nn/modules/dropout.py in __init__(self, p, inplace)
8 def __init__(self, p=0.5, inplace=False):
9 super(_DropoutNd, self).__init__()
---> 10 if p < 0 or p > 1:
11 raise ValueError("dropout probability has to be between 0 and 1, "
12 "but got {}".format(p))
RuntimeError: bool value of Tensor with more than one value is ambiguous
Why do you think there is an error?
Before putting the droprate, everything was working. Additional points for you if you know how to
implement a bias inside my network! For example, on the hidden layer. I can't find any example online.
Change your architecture for this:
class NeuralNet(nn.Module):
def __init__(self, input_size, hidden_size, num_classes, p=dropout):
super(NeuralNet, self).__init__()
self.fc1 = nn.Linear(input_size, hidden_size)
self.fc2 = nn.Linear(hidden_size, hidden_size)
self.fc3 = nn.Linear(hidden_size, num_classes)
self.dropout = nn.Dropout(p=p)
def forward(self, x):
out = F.relu(self.fc1(x))
out = F.relu(self.fc2(out))
out = self.dropout(self.fc3(out))
return out
Let me know if it works.
I've been trying to recreate a simple DNN using just the base Keras layer and writing everything from scratch. Everything seems to work just fine, but during the training loop I get this error:
AttributeError: 'SparseCategoricalCrossentropy' object has no attribute '_id'
I've tried changing the loss function to either CategoricalCrossentropy and SparseCategoricalCrossentropy (with from_logits True or False), but the error always pops up.
Here's the code:
import numpy as np
import tensorflow as tf
from tensorflow import keras
from utils import plot_image, plot_mnist_results, plot_value_array
class Flatten(keras.layers.Layer):
def __init__(self):
super(Flatten, self).__init__()
def build(self, input_shape):
self.output_size = np.prod(input_shape)
def call(self, X):
return tf.reshape(X, shape=(-1, self.output_size))
class Dense(keras.layers.Layer):
def __init__(self, units, activation):
super(Dense, self).__init__()
self.units = units
self.activation = activation
def build(self, input_shape):
self.kernel = self.add_weight(
name='kernel',
dtype=tf.float64,
initializer='glorot_normal',
trainable=True,
shape=(input_shape[-1], self.units)
)
self.bias = self.add_weight(
name='bias',
dtype=tf.float64,
initializer=keras.initializers.Constant(0.1),
trainable=True,
shape=(1, self.units)
)
def call(self, X):
return self.activation(tf.matmul(X, self.kernel) + self.bias)
class DNN(keras.models.Model):
def __init__(self, units, activation):
super(DNN, self).__init__()
self.units = units
self.activation = activation
def build(self, input_shape):
self.flatten = Flatten()
self.hidden_layer = Dense(self.units, tf.nn.relu)
self.output_layer = Dense(10, tf.nn.softmax)
def call(self, X):
print(self.hidden_layer(self.flatten(X)).shape)
print(self.output_layer(self.hidden_layer(self.flatten(X))).shape)
return self.output_layer(self.hidden_layer(self.flatten(X)))
# #tf.function
def train(model, loss, opt, X, y):
with tf.GradientTape() as tape:
gradients = tape.gradient(loss(model(X), y), model.trainable_variables)
gradient_variables = zip(gradients, model.trainable_variables)
opt.apply_gradients(gradient_variables)
mnist = keras.datasets.mnist
(train_images, train_labels), (test_images, test_labels) = mnist.load_data()
train_images = train_images / 255.0
test_images = test_images / 255.0
model = DNN(units=128, activation=tf.nn.relu)
opt = tf.optimizers.Adam(learning_rate=1e-3)
for epoch in range(3):
for step in range(train_labels.shape[0]):
loss = keras.losses.SparseCategoricalCrossentropy
train(model, loss, opt, train_images[step, :, :], train_labels[step])
train_loss = loss(model(train_images), train_labels)
template = 'Epoch {}, Train loss: {:.5f}'
print(template.format(epoch + 1, train_loss.numpy()))
I would expect for the model to train successfully, but it doesn't seem to be the case. What am I doing wrong?
From the given code, i could see that you are using tf and keras intermixed in places like given below.
opt = tf.optimizers.Adam(learning_rate=1e-3)
loss = keras.losses.SparseCategoricalCrossentropy
This could raise issues like this. For TensorFlow 2.0, you can use tf.keras uniformly in all places wherever you use keras directly.
Also i could find that, you are instantiating loss object inside the batch loop. which is not correct. You have to instantiate at the top of starting you epoch loop.
Rest all seems fine. Hope this helps!!!