We Keep Coding

How to get the filenames of misclassified images in the test data set - pytorch?

I am trying to use AlexNet to classify spectrogram images generated for 3s audio segments. I have successfully trained my dataset and am trying to identify which images the model misclassified.
I am able to obtain the filename of a image by calling iterator.dataset.data_df.adressfname, however I am unsure how to use this statement in the for loop in the get_predictions function. If I try to retrieve the filenames using iterator.dataset.data_df.adressfname[i], I get the following error: expected Tensor as element 0 in argument 0, but got str
Ultimately, I want to create a dataframe that contains the filename, actual label and predicted label. Does anyone have any suggestions?
class CustomDataset(Dataset):
def __init__(self, img_path, csv_file, transforms):
self.imgs_path = img_path
self.csv_train_file = csv_file
file_list = glob.glob(self.imgs_path + "*")
self.data = []
self.data_df = pd.read_csv(self.csv_train_file)
self.transforms = transforms
for ind in self.data_df.index:
img_path = self.data_df['spectrogramSegFilename'][ind]
class_name = self.data_df['dx'][ind]
self.data.append([img_path, class_name])
self.class_map = {"ProbableAD" : 0, "Control": 1}
self.img_dim = (256, 256)
def __len__(self):
return len(self.data)
def __getitem__(self, idx):
img_path, class_name = self.data[idx]
img = cv2.imread(img_path)
img = cv2.resize(img, self.img_dim)
class_id = self.class_map[class_name]
img_tensor = torch.from_numpy(img)
img_tensor = img_tensor.permute(2, 0, 1)
data = self.transforms(img_tensor)
class_id = torch.tensor([class_id])
return data, class_id
if __name__ == "__main__":
transformations = transforms.Compose([transforms.ToPILImage(), transforms.Resize(256), transforms.CenterCrop(256), transforms.ToTensor(), transforms.Normalize((0.49966475, 0.1840554, 0.34930056), (0.35317238, 0.17343724, 0.1894943))])
train_dataset = CustomDataset("/spectrogram_images/spectrogram_train/", "train_features_segmented.csv", transformations)
test_dataset = CustomDataset("/spectrogram_images/spectrogram_test/", "test_features_segmented.csv", transformations)
train_dataset = CustomDataset("spectrogram_images/spectrogram_train/", "/train_features_segmented.csv")
test_dataset = CustomDataset("spectrogram_images/spectrogram_test/", "/test_features_segmented.csv")
train_data_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)
test_data_loader = DataLoader(test_dataset, batch_size=64, shuffle=True)
def get_predictions(model, iterator, device):
model.eval()
images = []
labels = []
probs = []
participant_ids = []
with torch.no_grad():
for i, (x, y) in enumerate(iterator):
x = x.to(device)
y_pred = model(x)
y_prob = F.softmax(y_pred, dim=-1)
participant_ids.append(iterator.dataset.data_df.adressfname[i])
images.append(x.cpu())
labels.append(y.cpu())
probs.append(y_prob.cpu())
images = torch.cat(images, dim=0)
labels = torch.cat(labels, dim=0)
probs = torch.cat(probs, dim=0)
participant_ids = torch.cat(participant_ids, dim=0)
return images, labels, probs, participant_ids

Just add a third field to the return signature of __getitem__():
def __getitem__(self,idx):
...
return data,class_id,img_path
Then, when you call the dataloader:
for i, (x,y,img_paths) in enumerate(iterator):
... call model ...
... compare outputs to labels ...
... identify incorrect batch indices
mislabeled_files = [img_paths[idx] for idx in incorrect_batch_indices]
And there you have it. Note that in your current code block, i indexes the batch index, which really has nothing to do with the dataset object wrapped in the dataloader (both because the dataloader has a batch size so it collates multiple elements from the dataset into a single index, and because the dataloader shuffles the elements in the dataset) so you should not use this index (i) to reference the dataset object. If you wanted to reference the underlying data items in the dataset object you could instead simply return the data idx in __getitem__():
def __getitem__(self,idx):
...
return data,class_id,idx
You can then use this idx to reference the original dataset data and get the file names that way.

Nested Cross Validation in PyTorch

I am working on a multiclass image classification problem that has 4k labeled images. Currently, I am using cross-validation.However, I want to try nested-cross-validation for the problem in pytorch but I couldn't find it.
Is it possible to use nested-cross-validation in pytorch? if so, how?
I built a simple pipeline below. Is it correct to implement nested-cross-validation like this.
import torch
from torch.utils.data import DataLoader,SubsetRandomSampler
from sklearn.model_selection import KFold
from torchvision import datasets
input_size = (256,3,224,244)
target_size = (256,)
class Dataset(datasets.VisionDataset):
def __init__(self):
super().__init__(self)
self.images = torch.rand(input_size).float()
self.targets = torch.randint(0,3,target_size)
def __getitem__(self, index: int) -> any:
return self.images[index],self.targets[index]
def __len__(self) -> int:
return len(self.images)
class BasicModel(torch.nn.Module):
def __init__(self) -> None:
super(BasicModel,self).__init__()
self.conv = torch.nn.Conv2d(3,16,kernel_size=(5,5))
self.adp = torch.nn.AdaptiveAvgPool2d(1)
self.linear = torch.nn.Linear(16,3)
def forward(self,x):
x = self.conv(x)
x = self.adp(x)
x = x.view(x.size(0),-1)
x = self.linear(x)
return x
data_ids = [*range(input_size[0])]
data = Dataset()
model = BasicModel()
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(),lr=0.01)
k_fold = 5
kfold_test = KFold(n_splits=k_fold, shuffle=True)
num_epochs = 2
for test_fold,(remain_ids,test_ids) in enumerate(kfold_test.split(data_ids)):
test_sampler = SubsetRandomSampler(test_ids)
testLoader = DataLoader(data,sampler=test_sampler,batch_size=2)
kfold_val = KFold(n_splits=k_fold-1, shuffle=True)
for epoch in range(num_epochs):
for val_fold,(train_ids,val_ids) in enumerate(kfold_val.split(remain_ids)):
train_sampler = SubsetRandomSampler(train_ids)
trainLoader = DataLoader(data,sampler=train_sampler,batch_size=2)
val_sampler = SubsetRandomSampler(val_ids)
valLoader = DataLoader(data,sampler=val_sampler,batch_size=2)
model.train()
for image,target in trainLoader:
with torch.cuda.amp.autocast():
output = model(image)
loss = criterion(output,target)
optimizer.zero_grad()
loss.backward()
optimizer.step()
print(f"train loss:{loss.item()}")
model.eval()
with torch.no_grad():
for image,target in valLoader:
output = model(image)
loss = criterion(output,target)
print(f"test loss:{loss.item()}")
model.eval()
with torch.no_grad():
for image,target in testLoader:
output = model(image)
loss = criterion(output,target)
print(f"test loss:{loss.item()}")

How to use pre-trained models for text classification？Comparing a fine-tuned model with a pre-trained model without fine-tuning

I want to know how much the fine-tuned model improves compared to the model without fine-tuning.I want to compare the performance of the pre-trained model(BERT) and the model(fine-tuned BERT) obtained by fine-tuning the pre-trained model on text classification.I know how to fine-tune BERT for text classification, but not very clear on how to use BERT directly for classification.what should I do?The following is the code for fine-tuning the model, how to rewrite it to directly use the pre-trained model.
<!-- language: python -->
from transformers import BertTokenizer, BertModel
import torch
import torch.nn as nn
import torch.utils.data as Data
import torch.optim as optim
from sklearn.metrics import accuracy_score,matthews_corrcoef
from sklearn.model_selection import train_test_split
tokenizer_model = BertTokenizer.from_pretrained('bert-base-uncased')
pretrained_model = BertModel.from_pretrained("bert-base-uncased")
class MyDataSet(Data.Dataset):
def __init__ (self, data, label):
self.data = data
self.label = label
self.tokenizer = tokenizer_model
def __getitem__(self, idx):
text = self.data[idx]
label = self.label[idx]
inputs = self.tokenizer(text, return_tensors="pt",padding='max_length',max_length=256,truncation=True)
input_ids = inputs.input_ids.squeeze(0)
#token_type_ids = inputs.token_type_ids.squeeze(0)
attention_mask = inputs.attention_mask.squeeze(0)
#return input_ids, token_type_ids, attention_mask, label
return input_ids, attention_mask, label
def __len__(self):
return len(self.data)
data,label = [],[]
with open(path) as f:
for line in f.readlines():
a,b = line.strip().split('\t')
data.append(b)
if a == 'LOW':
label.append('0')
elif a == 'MEDIUM':
label.append('1')
else:
label.append('2')
label = [int(i) for i in label]
train_x,test_x,train_y,test_y = train_test_split(data, label, test_size = 0.15,random_state = 32, stratify=label)
dataset_train = MyDataSet(train_x,train_y)
dataset_test = MyDataSet(test_x,test_y)
dataloader_train = Data.DataLoader(dataset_train, batch_size=128, shuffle=True,num_workers=32,pin_memory=True)
dataloader_test = Data.DataLoader(dataset_test, batch_size=128, shuffle=True,num_workers=32,pin_memory=True)
class MyModel(nn.Module):
def __init__(self):
super(MyModel, self).__init__()
self.bert = pretrained_model
self.linear = nn.Linear(768,3)
def forward(self, input_ids, attention_mask):
output = self.bert(input_ids, attention_mask).pooler_output
print(output.shape) # torch.Size([1, 768])
output = self.linear(output)
return output
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if torch.cuda.device_count() > 1:
print("Use", torch.cuda.device_count(), 'gpus')
model = MyModel()
model = nn.DataParallel(model)
model = model.to(device)
## model = MyModel().to(device)
loss_fn = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=1e-5)
for epoch in range(10):
for input_ids,attention_mask,label in dataloader_train:
train_input_ids,train_attention_mask,train_label = input_ids.to(device),attention_mask.to(device),label.to(device)
model.train()
pred = model(train_input_ids,train_attention_mask)
print('epoch:',epoch)
#print('pred,label:',pred,label)
loss = loss_fn(pred, train_label)
print('Loss:',loss.item())
pred = torch.argmax(pred,dim=1)
acc = (pred == train_label).float().mean()
print('acc:',acc)
loss.backward()
optimizer.step()
optimizer.zero_grad()
savename_train = str(path) +'_' + str(name) + '_train' + '.txt'
with open(savename_train,'a') as f:
f.write(str(epoch)+'\t'+str(loss.item())+'\t'+str(acc.item())+'\n')
model.eval()
with torch.no_grad():
for input_ids,attention_mask,label in dataloader_test:
validation_input_ids,validation_attention_mask,validation_label = input_ids.to(device),attention_mask.to(device),label.to(device)
pred = model(validation_input_ids,validation_attention_mask)
loss = loss_fn(pred, validation_label)
pred = torch.argmax(pred, dim=1)
acc = (pred == validation_label).float().mean()
print('acc:',acc)
savename_eval = str(path) +'_' + str(name) + '_val' + '.txt'
with open(savename_eval,'a') as f:
f.write(str(epoch)+'\t'+str(loss.item())+'\t'+str(acc.item())+'\n')

What you are trying to do does not make sense. The naive BERT model was retrained using a combination of masked language modelling objective and next sentence prediction. So, all it can do is predicting masked tokens, predicting if a pair of given sentence can be next to each other in a text. Most importantly, it can provide embeddings.
To use for classification you have to add a classification head to the end of the model. Initially, the weights of that layer is randomly initialised. If you do not fine tune the last layer, what do you really expect from random weights?
If you really want to compare the fine-tuned model to a baseline, take the embeddings vector from the BERT and use a tradional ML model like SVM or Tree based calssifier.

tensorboard,, the scalar value is not recorded

The scalar value is not being recorded as in these pictures.
enter image description here 2.
enter image description here
I don't know if I'm doing the wrong with code for training or code for tensorboard.
It's my first time using a tensorboard, so I don't know if I wrote the correct code for viewing the tensorboard.
If the scalar value is not recorded as shown in the attached image even though the code for the tensorboard is written correctly, I would like to look at the code part for training.
Therefore, I would appreciate it if you could check code only from recording the scalar value to verifying it with the tensorboard
part of the main function of <run.py>
def main():
args = parser.parse_args()
assert args.n_views == 2, "Only two view training is supported. Please use --n-views 2."
# check if gpu training is available
if not args.disable_cuda and torch.cuda.is_available():
args.device = torch.device('cuda')
cudnn.deterministic = True
cudnn.benchmark = True
else:
args.device = torch.device('cpu')
args.gpu_index = -1
dataset = ContrastiveLearningDataset(args.data)
train_dataset = dataset.__getitem__(args.dataset_name, args.n_views, 1)
train_loader = torch.utils.data.DataLoader(
train_dataset, batch_size=args.batch_size, shuffle=True,
num_workers=args.workers, pin_memory=True, drop_last=True)
#f(*)
model = ResNetSimCLR(base_model=args.arch, out_dim=args.out_dim)
optimizer = torch.optim.Adam(model.parameters(), args.lr, weight_decay=args.weight_decay)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=len(train_loader), eta_min=0,
last_epoch=-1)
with torch.cuda.device(args.gpu_index):
simclr = SimCLR(model=model, optimizer=optimizer, scheduler=scheduler, args=args)
simclr.train(train_loader)
simclr.writer.flush()
<simclr.py> to write train function in main()
class SimCLR(object):
def __init__(self, *args, **kwargs):
self.args = kwargs['args']
self.model = kwargs['model'].to(self.args.device)
self.optimizer = kwargs['optimizer']
self.scheduler = kwargs['scheduler']
self.writer = SummaryWriter()
logging.basicConfig(filename=os.path.join(self.writer.log_dir, 'training.log'), level=logging.DEBUG)
self.criterion = torch.nn.CrossEntropyLoss().to(self.args.device)
def info_nce_loss(self, features):
labels = torch.cat([torch.arange(self.args.batch_size) for i in range(self.args.n_views)], dim=0)
labels = (labels.unsqueeze(0) == labels.unsqueeze(1)).float()
labels = labels.to(self.args.device)
features = F.normalize(features, dim=1)
similarity_matrix = torch.matmul(features, features.T)
mask = torch.eye(labels.shape[0], dtype=torch.bool).to(self.args.device)
labels = labels[~mask].view(labels.shape[0], -1)
similarity_matrix = similarity_matrix[~mask].view(similarity_matrix.shape[0], -1)
positives = similarity_matrix[labels.bool()].view(labels.shape[0], -1)
negatives = similarity_matrix[~labels.bool()].view(similarity_matrix.shape[0], -1)
logits = torch.cat([positives, negatives], dim=1)
labels = torch.zeros(logits.shape[0], dtype=torch.long).to(self.args.device)
logits = logits / self.args.temperature
return logits, labels
def train(self, train_loader):
scaler = GradScaler(enabled=self.args.fp16_precision)
# save config file
save_config_file(self.writer.log_dir, self.args)
n_iter = 0 #global optimization step
logging.info(f"Start SimCLR training for {self.args.epochs} epochs.")
logging.info(f"Training with gpu: {self.args.disable_cuda}.")
for epoch_counter in range(self.args.epochs):
for images, _ in tqdm(train_loader):
images = torch.cat(images, dim=0)
images = images.to(self.args.device)
with autocast(enabled=self.args.fp16_precision):
features = self.model(images)
logits, labels = self.info_nce_loss(features)
loss = self.criterion(logits, labels)
self.optimizer.zero_grad()
scaler.scale(loss).backward()
scaler.step(self.optimizer)
scaler.update()
if n_iter % self.args.log_every_n_steps == 0:
top1, top5 = accuracy(logits, labels, topk=(1, 5))
self.writer.add_scalar('loss', loss, global_step=n_iter)
self.writer.add_scalar('acc/top1', top1[0], global_step=n_iter)
self.writer.add_scalar('acc/top5', top5[0], global_step=n_iter)
self.writer.add_scalar('learning_rate', self.scheduler.get_lr()[0], global_step=n_iter)
n_iter += 1
# warmup for the first 10 epochs
if epoch_counter >= 1:
try:
self.scheduler.step()
except ZeroDivisionError:
print("ZeroDivision")
logging.debug(f"Epoch: {epoch_counter}\tLoss: {loss}\tTop1 accuracy: {top1[0]}")
logging.info("Training has finished.")
# save model checkpoints
checkpoint_name = 'checkpoint_{:04d}.pth.tar'.format(self.args.epochs)
save_checkpoint({
'epoch': self.args.epochs,
'arch': self.args.arch,
'state_dict': self.model.state_dict(),
'optimizer': self.optimizer.state_dict(),
}, is_best=False, filename=os.path.join(self.writer.log_dir, checkpoint_name))
logging.info(f"Model checkpoint and metadata has been saved at {self.writer.log_dir}.")

Writing summary.scalar with Dataset API and Keras

I use tensorflow Keras API and try to add custom scalar to the tensorboard but nothing except the loss is displayed.
Here is the code for the model:
embedding_in = Embedding(
input_dim=vocab_size + 1 + 1,
output_dim=dim,
mask_zero=True,
)
embedding_out = Embedding(
input_dim=vocab_size + 1 + 1,
output_dim=dim,
mask_zero=True,
)
input_a = Input((None,))
input_b = Input((None,))
input_c = Input((None, None))
emb_target = embedding_in(input_a)
emb_context = embedding_out(input_b)
emb_negatives = embedding_out(input_c)
emb_gru = GRU(dim, return_sequences=True)(emb_target)
num_negatives = tf.shape(input_c)[-1]
def make_logits(tensors):
emb_gru, emb_context, emb_negatives = tensors
true_logits = tf.reduce_sum(tf.multiply(emb_gru, emb_context), axis=2)
true_logits = tf.expand_dims(true_logits, -1)
sampled_logits = tf.squeeze(
tf.matmul(emb_negatives, tf.expand_dims(emb_gru, axis=2),
transpose_b=True), axis=3)
true_logits = true_logits*0
sampled_logits = sampled_logits*0
logits = K.concatenate([true_logits, sampled_logits], axis=-1)
return logits
logits = Lambda(make_logits)([emb_gru, emb_context, emb_negatives])
mean = tf.reduce_mean(logits)
tf.summary.scalar('mean_logits', mean)
model = keras.models.Model(inputs=[input_a, input_b, input_c], outputs=[logits])
In particular, I want to see the evolution of mean_logits scalar after each batch.
I create and compile the model like this:
model = build_model(dim, vocab_size)
model.compile(loss='binary_crossentropy', optimizer='sgd')
callbacks = [
keras.callbacks.TensorBoard(logdir, histogram_freq=1)
]
I use tf Dataset API to the model:
iterator = dataset.make_initializable_iterator()
with tf.Session() as sess:
sess.run(iterator.initializer)
sess.run(tf.tables_initializer())
model.fit(iterator, steps_per_epoch=100,
callbacks=callbacks,
validation_data=iterator,
validation_steps=1
)
However, I don't get any mean_logits graph in the tensorboard and it's not in the graphs.
How can I track mean_logits scalar in tensorboard after each batch?
I use tf 1.12 and keras 2.1.

I have also faced the same issue. It seems that Keras TensorBoard callback not gonna write all existing summaries automatically, but only those registered as metrics (and appear in logs dict). Updating the logs object is a nice trick as it allows to use the values in other callbacks, see Early stopping and learning rate schedule based on custom metric in Keras. I can see several possibilities:
1. Using Lambda callback
Something like this:
eval_callback = LambdaCallback(
on_epoch_end=lambda epoch, logs: logs.update(
{'mean_logits': K.eval(mean)}
))
2. Custom TensorBoard callback
You can also subclass the callback and define your own logic. For instance, my workaround for learning rate monitoring:
class Tensorboard(Callback):
def __init__(self,
log_dir='./log',
write_graph=True):
self.write_graph = write_graph
self.log_dir = log_dir
def set_model(self, model):
self.model = model
self.sess = K.get_session()
if self.write_graph:
self.writer = tf.summary.FileWriter(self.log_dir, self.sess.graph)
else:
self.writer = tf.summary.FileWriter(self.log_dir)
def on_epoch_end(self, epoch, logs={}):
logs.update({'learning_rate': float(K.get_value(self.model.optimizer.lr))})
self._write_logs(logs, epoch)
def _write_logs(self, logs, index):
for name, value in logs.items():
if name in ['batch', 'size']:
continue
summary = tf.Summary()
summary_value = summary.value.add()
if isinstance(value, np.ndarray):
summary_value.simple_value = value.item()
else:
summary_value.simple_value = value
summary_value.tag = name
self.writer.add_summary(summary, index)
self.writer.flush()
def on_train_end(self, _):
self.writer.close()
Here, I just add 'learning_rate' to logs explicitly. But this way can be much more flexible and powerful.
3. Metrics trick
Here is another interesting workaround. What you need to do is to pass a custom metric function to model's compile() call which returns aggregated summary tensor. The idea is to make Keras pass your aggregated summary operation to every session.run call and return it's result as metric:
x_entropy_t = K.sum(p_t * K.log(K.epsilon() + p_t), axis=-1, keepdims=True)
full_policy_loss_t = -res_t + X_ENTROPY_BETA * x_entropy_t
tf.summary.scalar("loss_entropy", K.sum(x_entropy_t))
tf.summary.scalar("loss_policy", K.sum(-res_t))
tf.summary.scalar("loss_full", K.sum(full_policy_loss_t))
summary_writer = tf.summary.FileWriter("logs/" + args.name)
def summary(y_true, y_pred):
return tf.summary.merge_all()
value_policy_model.compile(optimizer=Adagrad(), loss=loss_dict, metrics=[summary])
l = value_policy_model.train_on_batch(x_batch, y_batch)
l_dict = dict(zip(value_policy_model.metrics_names, l))
summary_writer.add_summary(l_dict['value_summary'], global_step=iter_idx)
summary_writer.flush()