I am using the following packages in my project:
boto3==1.20.19
flashtorch==0.1.3
matplotlib==3.3.4
numpy==1.13.3
Pillow==8.4.0
scikit_learn==1.0.1
scipy==1.5.4
seaborn==0.11.2
torch==1.10.0
torchvision==0.11.1
When I try to execute the following dependencies:
import seaborn as sns
import copy
import boto3
from scipy.stats import spearmanr
import random
import csv
from sklearn.model_selection import train_test_split
import copy
import time
import numpy as np
import os
import pickle
import torch
from torchvision import transforms
import torchvision.models as models
from torch.utils import data
import matplotlib.pyplot as plt
from PIL import Image
from sklearn.metrics import f1_score
from scipy.stats import pearsonr
from flashtorch.saliency import Backprop
from scipy.ndimage.filters import gaussian_filter
def split_train_test(test_size=0.1):
# Make partition directory.
print("make partition directory if not exists")
partition_p = os.path.join("..", "data", "partition")
if not os.path.exists(partition_p):
os.mkdir(partition_p)
# List all input filenames and shuffle.
base_p = os.path.join("..", "data", "imgs")
img_ps = [] # e.g. ../data/imgs/MINT/foo.jpg.
for dir_name in os.listdir(base_p):
dir_p = os.path.join(base_p, dir_name)
for img_fname in os.listdir(dir_p):
img_p = os.path.join(dir_name, img_fname)
img_ps.append(img_p)
random.shuffle(img_ps)
# Keep only baseball cards in MINT or POOR condition, for now.
print("Keep only baseball cards, for now")
baseball_sport = "185223"
img_ps = [p for p in img_ps if baseball_sport in p]
# Keep only desired conditions.
print("Exclude good and fair cards to create a 9, 7, 5, 3, 1 scale")
img_ps = [p for p in img_ps if "GOOD" not in p and "FAIR" not in p]
# Remove "bad" images.
# Try to load each image, and only retain it if we can load it
# successfully as a (255, 255, 3) image.
print("remove 'bad' images")
old_img_ps = copy.copy(img_ps)
img_ps = []
counter = 0
for p in old_img_ps:
try:
# counter.
counter += 1
if counter % 100 == 0:
print(counter)
# validate image.
full_p = os.path.join("..", "data", "imgs", p)
img = Image.open(full_p)
img = np.array(img)
assert img.shape[2] == 3
# add image to paths.
img_ps.append(p)
except Exception as e:
print("Skip bad image {}".format(p))
print(e)
# Train/test split.
print("split train, test")
train_ps, test_ps = train_test_split(img_ps, test_size=test_size)
# Create partition dictionary.
print("create partition object")
partition = {
"train": train_ps,
"test": test_ps
}
# Pickle partition object.
of_p = os.path.join(partition_p, "partition.p")
with open(of_p, "wb") as of:
pickle.dump(partition, of)
print("wrote {}".format(of_p))
# Map all unique labels to integer IDs.
# lbl_to_idx is a dictinary mapping e.g. "EX" -> 0, etc.
# str_labels = ["MINT", "NM", "EX", "VG", "GOOD", "FAIR", "POOR"]
str_labels = ["MINT", "NM", "EX", "VG", "POOR"]
lbl_to_idx = {lbl: i for i, lbl in enumerate(str_labels)}
# Create labels dictionary.
# fname_to_lbl is a dictionary mapping e.g. "some_fname.npy" -> 2
# Example filename.100097_sport185226_conditionVG_preprocessed.npy
fname_to_lbl = {}
for p in img_ps:
# Extract label.
str_lbl = p.split("_")[-1].lstrip("condition").rstrip(".jpg")
int_lbl = lbl_to_idx[str_lbl]
# Add to dictionary.
fname_to_lbl[p] = int_lbl
# Pickle fname_to_lbl mapping.
of_p = os.path.join(partition_p, "labels.p")
with open(of_p, "wb") as of:
pickle.dump(fname_to_lbl, of)
print("wrote {}".format(of_p))
class Dataset(data.Dataset):
def __init__(self, list_IDs, labels, apply_rotations=False):
"""
parameters
----------
list_IDs (str []) file paths.
labels (str []) target labels.
apply_rotations (bool) : if True then randomly rotate images.
NOTE! This should be FALSE for testing data.
"""
self.labels = labels # dictionary. key=fname, value=integer lbl.
self.list_IDs = list_IDs # list of filenames.
self.apply_rotations = apply_rotations
self.rotation_fn = transforms.Compose([
transforms.RandomAffine(degrees=90, shear=90)
])
# self.rotation_fn = transforms.RandomRotation(90) # torch rotation function.
# Initialize AWS storage (s3) resource.
self.s3_resource = boto3.resource("s3")
# Make ML_images directory and subdirectories.
conditions = ["MINT", "NM", "EX", "VG", "FAIR", "GOOD", "POOR"]
base_p = os.path.join("..", "data", "ml_images")
if not os.path.exists(base_p):
os.mkdir(base_p)
for condition in conditions:
p = os.path.join(base_p, condition)
os.mkdir(p)
def __len__(self):
return len(self.list_IDs)
def download_from_s3(self, remote_fname):
# download fname from s3 as ../data/ml_images/{REMOTE FNAME}
local_fname = os.path.join("..", "data", "ml_images", remote_fname)
print("download {}".format(remote_fname))
# download image from S3 as "img.jpg"
resp = self.s3_resource.Object(
"mintcondition",
remote_fname
).download_file(local_fname)
return None
def load_and_preprocess_img(self, img_p, apply_rotations=False):
""" load and preprocess img, optionally applying rotations
parameters:
-----------
img_p (str) path to image.
apply_rotations (bool) if True apply rotations to the image.
should be false to all testing.
returns:
--------
X (torch tensor) Image as torch tensor of shape (3, 255, 255)
"""
# Load image and reshape to (3, 255, 255)
img = Image.open(img_p)
img = img.resize((255, 255), Image.ANTIALIAS)
# optionally rotate.
if apply_rotations:
img = self.rotation_fn(img)
# Cast to torch tensor.
X = np.array(img) # (255, 255, 3) numpy
assert X.shape == (255, 255, 3)
X = X / 255 # "normalize"
X = X.swapaxes(1, 2) # (255, 3, 255)
X = X.swapaxes(0, 1) # (3, 255, 255) numpy
X = torch.from_numpy(X).float() # (3, 255, 255) torch
return X
def __getitem__(self, index):
try:
# Select sample.
remote_p = self.list_IDs[index]
# Get label.
y = self.labels[remote_p]
# If the image does not exist locally, get it from S3.
local_p = os.path.join("..", "data", "ml_images", remote_p)
if not os.path.exists(local_p):
self.download_from_s3(remote_p)
# Load image and reshape to torch tensor of shape (3, 255, 255)
X = self.load_and_preprocess_img(local_p, apply_rotations=self.apply_rotations) # torch tensor.
except Exception as e:
print(e)
print("exception loading data..using random image, label instead")
X = np.random.random((3, 255, 255))
X = torch.from_numpy(X).float()
y = 0
return X, y
def set_parameter_requires_grad(model, feature_extracting):
if feature_extracting:
for param in model.parameters():
param.requires_grad = False
def reenable_gradients(model):
for param in model.parameters():
param.requires_grad = True
def initialize_model(model_name, num_classes, feature_extract, use_pretrained=True):
model_ft = None
input_size = 0
if model_name == "resnet":
""" Resnet18
"""
model_ft = models.resnet18(pretrained=use_pretrained)
set_parameter_requires_grad(model_ft, feature_extract)
num_ftrs = model_ft.fc.in_features
model_ft.fc = torch.nn.Linear(num_ftrs, num_classes)
input_size = 224
elif model_name == "alexnet":
""" Alexnet
"""
model_ft = models.alexnet(pretrained=use_pretrained)
set_parameter_requires_grad(model_ft, feature_extract)
num_ftrs = model_ft.classifier[6].in_features
model_ft.classifier[6] = torch.nn.Linear(num_ftrs, num_classes)
input_size = 224
elif model_name == "vgg":
model_ft = models.vgg11_bn(pretrained=use_pretrained)
set_parameter_requires_grad(model_ft, feature_extract)
num_ftrs = model_ft.classifier[6].in_features
model_ft.classifier[6] = nn.Linear(num_ftrs, num_classes)
input_size = 224
elif model_name == "squeezenet":
model_ft = models.squeezenet1_0(pretrained=use_pretrained)
set_parameter_requires_grad(model_ft, feature_extract)
model_ft.classifier[1] = nn.Conv2d(512, num_classes, kernel_size=(1, 1), stride=(1, 1))
model_ft.num_classes = num_classes
input_size = 224
elif model_name == "densenet":
model_ft = models.densenet121(pretrained=use_pretrained)
set_parameter_requires_grad(model_ft, feature_extract)
num_ftrs = model_ft.classifier.in_features
model_ft.classifier = nn.Linear(num_ftrs, num_classes)
input_size = 224
elif model_name == "inception":
""" Inception v3
Be careful, expects (299,299) sized images and has auxiliary output
"""
model_ft = models.inception_v3(pretrained=use_pretrained)
set_parameter_requires_grad(model_ft, feature_extract)
# Handle the auxilary net
num_ftrs = model_ft.AuxLogits.fc.in_features
model_ft.AuxLogits.fc = nn.Linear(num_ftrs, num_classes)
# Handle the primary net
num_ftrs = model_ft.fc.in_features
model_ft.fc = nn.Linear(num_ftrs, num_classes)
input_size = 299
else:
print("Invalid model name, exiting...")
exit()
return model_ft, input_size
def train_model(model, dataloaders, criterion, optimizer, num_epochs=25, is_inception=False, device="cpu"):
since = time.time()
val_acc_history = []
best_model_wts = copy.deepcopy(model.state_dict())
best_acc = 0.0
# Initiailize a file to track accuracy over epochs.
acc_of_p = os.path.join("..", "data", "model_accuracy.csv")
acc_of = open(acc_of_p, "w", newline="")
header = ["epoch", "phase", "accuracy", "F", "r"]
w = csv.writer(acc_of)
w.writerow(header)
for epoch in range(num_epochs):
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 10)
# Each epoch has a training and validation phase
for phase in ['train', 'test']:
if phase == 'train':
model.train() # Set model to training mode
else:
model.eval() # Set model to evaluate mode
running_loss = 0.0
running_corrects = 0.0
ypreds = []
ytrues = []
# Iterate over data.
batch_num = 0
for inputs, labels in dataloaders[phase]:
inputs = inputs.to(device)
labels = labels.to(device)
# zero the parameter gradients
optimizer.zero_grad()
# forward
# track history if only in train
with torch.set_grad_enabled(phase == 'train'):
# Get model outputs and calculate loss
# Special case for inception because in training it has an auxiliary output. In train
# mode we calculate the loss by summing the final output and the auxiliary output
# but in testing we only consider the final output.
if is_inception and phase == 'train':
# From https://discuss.pytorch.org/t/how-to-optimize-inception-model-with-auxiliary-classifiers/7958
outputs, aux_outputs = model(inputs)
loss1 = criterion(outputs, labels)
loss2 = criterion(aux_outputs, labels)
loss = loss1 + 0.4 * loss2
else:
outputs = model(inputs)
loss = criterion(outputs, labels)
_, preds = torch.max(outputs, 1)
# backward + optimize only if in training phase
if phase == 'train':
loss.backward()
optimizer.step()
# statistics
running_loss += loss.item() * inputs.size(0)
running_corrects += torch.sum(preds == labels.data)
F = f1_score(preds.numpy(), labels.data.numpy(), average="micro")
ypreds.extend(list(preds.numpy()))
ytrues.extend(list(labels.data.numpy()))
# counter.
batch_num += 1
if batch_num % 1 == 0:
correct = float(torch.sum(preds == labels.data))
incorrect = float(torch.sum(preds != labels.data))
perc_correct = 100 * correct / (correct + incorrect)
msg = """
epoch {} batch {} : percent correct={:.4f} F={:.4f}
""".format(epoch, batch_num, perc_correct, F)
print(msg)
print(preds)
print(labels.data)
# rank correlation of predicted, actual.
rho, p = spearmanr(preds.numpy(), labels.data.numpy())
print("correlation of pred, actual: rho = {:.4f}".format(rho))
epoch_loss = running_loss / len(dataloaders[phase].dataset)
epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)
epoch_F = f1_score(ypreds, ytrues, average="micro")
epoch_R = pearsonr(ypreds, ytrues)[0]
print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))
# deep copy the model
if phase == 'val' and epoch_acc > best_acc:
best_acc = epoch_acc
best_model_wts = copy.deepcopy(model.state_dict())
if phase == 'val':
val_acc_history.append(epoch_acc)
# Write latest train and test accuracies to output file.
out = [epoch, phase, epoch_acc.numpy(), epoch_F, epoch_R]
w.writerow(out)
acc_of.flush()
# Pickle the model after end of epoch.
of_p = os.path.join("..", "models", "latest_model.p")
with open(of_p, "wb") as of:
pickle.dump(model, of)
print("wrote {}".format(of_p))
time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
print('Best val Acc: {:4f}'.format(best_acc))
# close output file.
of.flush()
of.close()
print("wrote {}".format(acc_of_p))
# load best model weights
model.load_state_dict(best_model_wts)
return model, val_acc_history
def train_CNN_model(num_classes=5, load_latest_model=False):
# Use CUDA if available.
use_cuda = torch.cuda.is_available()
print("check if CUDA can be used -> {}".format(use_cuda))
device = torch.device("cuda:0" if use_cuda else "cpu")
# Define transformations of the images.
# Note that the normalization transform with these specific
# Parameters is necessary for working with pretrained AlexNet.
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
img_transforms = transforms.Compose([
normalize
])
# Parameters
params = {
"batch_size": 64,
"shuffle": True,
"num_workers": 4
}
max_epochs = 10
# Datasets.
print("load train/test partition and labels")
partition_p = os.path.join("..", "data", "partition", "partition.p")
labels_p = os.path.join("..", "data", "partition", "labels.p")
partition = pickle.load(open(partition_p, "rb"))
labels = pickle.load(open(labels_p, "rb"))
print("create generators over train, test data")
train_set = Dataset(partition["train"], labels, apply_rotations=True) # do rotate train data.
train_g = data.DataLoader(train_set, **params)
test_set = Dataset(partition["test"], labels, apply_rotations=False) # don't rotate test data.
test_g = data.DataLoader(test_set, **params)
if load_latest_model:
print("load pickled latest_model.p")
model_p = os.path.join("..", "models", "latest_model.p")
model = pickle.load(open(model_p, "rb"))
else:
model, _ = initialize_model(
model_name="resnet", # resnet 18 with skip connections.
num_classes=num_classes,
feature_extract=True, # if True only finetune top layer.
use_pretrained=True
)
model.to(device)
# Initialize optimizer, loss criterion.
optimizer = torch.optim.Adam(model.parameters())
criterion = torch.nn.CrossEntropyLoss()
# Train model.
model, history = train_model(
model=model,
dataloaders={"train": train_g, "test": test_g},
criterion=criterion,
optimizer=optimizer,
num_epochs=max_epochs,
is_inception=False,
device=device
)
# Pickle best performing model.
of_p = os.path.join("..", "models", "best_model.p")
with open(of_p, "wb") as of:
pickle.dump(model, of)
print("wrote {}".format(of_p))
# Pickle history of best performing model.
of_p = os.path.join("..", "models", "history.p")
with open(of_p, "wb") as of:
pickle.dump(history, of)
print("wrote {}".format(of_p))
def load_and_preprocess_img(img_p):
# Load image and reshape to (3, 255, 255)
img = Image.open(img_p)
img = img.resize((255, 255), Image.ANTIALIAS)
# Cast to torch tensor.
X = np.array(img) # (255, 255, 3) numpy
X = X.reshape((1, 255, 255, 3))
X = X / 255 # "normalize"
X = X.swapaxes(2, 3) # (64, 255, 3, 255)
X = X.swapaxes(1, 2) # (64, 3, 255, 255) numpy
X = torch.from_numpy(X).float() # (64, 3, 255, 255) torch
return X
def saliency_map(model, img_p):
# Load and preprocess image.
X = load_and_preprocess_img(img_p) # (3, 255, 255 torch tensor)
X.requires_grad_() # This is critical to actually get gradients.
with torch.set_grad_enabled(True):
backprop = Backprop(model) # flashtorch.saliency Backprop object.
gradients = backprop.calculate_gradients(X, take_max=True, guided=False) # (1, 255, 255)
# Cast image, saliency maps to numpy arrays.
X = X.detach() # must 'detach' from gradients before slicing.
img_np = X.numpy()[0] # (3, 255, 255)
img_np = img_np.swapaxes(0, 1) # (255, 3, 255)
img_np = img_np.swapaxes(1, 2) # (255, 255, 3)
saliency_map_np = gradients.numpy()[0] # (255, 255)
print(max(np.max(saliency_map_np, axis=0)))
print(saliency_map_np)
print(img_np.shape)
print(saliency_map_np.shape)
# Smooth heatmap.
saliency_map_np = gaussian_filter(saliency_map_np, sigma=10)
# Plot image and overlay saliency map.
heatmap = sns.heatmap(saliency_map_np, alpha=0.5)
heatmap.imshow(img_np, cmap="YlGnBu")
plt.show()
return saliency_map_np
if __name__ == "__main__":
# split_train_test()
# train_CNN_model(load_latest_model=False, num_classes=5)
model_p = os.path.join("..", "models", "cloud_model.p")
model = pickle.load(open(model_p, "rb"))
reenable_gradients(model) # e.g., disable fine-tuning mode
# img_p = os.path.join("..", "demos", "demo_cards",
# "very_good.jpg")
img_p = os.path.join("..", "data", "imgs", "POOR",
"9078_sport185223_conditionPOOR.jpg")
sm = saliency_map(model, img_p)
I get the following error:
Traceback (most recent call last):
File "/home/admin/Desktop/Code/project/venv/lib/python3.6/site-packages/torchvision/_internally_replaced_utils.py", line 14, in <module>
from torch.hub import load_state_dict_from_url
ModuleNotFoundError: No module named 'torch.hub'
Any suggestions why? What is wrong with my installed packages?
I appreciate your replies!
Related
I'm running a Kaggle Notebook, but when I'm running my training loop it shows GPU usage as Zero. When I run my training loop I print the line:
print("Using", torch.cuda.get_device_name(0))
Which returns:
Using Tesla P100-PCIE-16GB
But when I click the meters up at the top it shows CPU at 189% and GPU at 0%? Why would this show Zero if GPU is enabled?
Below is my DataLoader, Model, and Training Loop:
Dataloader:
def read_xray(file_path, img_size=None):
dicom = pydicom.read_file(file_path)
img = dicom.pixel_array
if dicom.PhotometricInterpretation == "MONOCHROME1":
img = np.max(img) - img
if img_size:
img_size = (img_size, img_size) # Convert img_size to a tuple
img = cv2.resize(img, dsize=img_size)
# Add channel dim at First
img = img[np.newaxis]
# Converting img to float32
img = img / np.max(img)
img = img.astype("float32")
return img
class ImageDataset(Dataset):
def __init__(self, root_dir, df, transform=None):
self.root_dir = root_dir
self.transform = transform
self.folder_list = os.listdir(self.root_dir)
self.image_counts = [len(os.listdir(os.path.join(self.root_dir, folder))) for folder in self.folder_list]
self.df = df
def __len__(self):
return sum(self.image_counts)
def __getitem__(self, index):
row = self.df.iloc[index]
folder_index = 0
while index >= self.image_counts[folder_index]:
index -= self.image_counts[folder_index]
folder_index += 1
image_path = os.path.join(self.root_dir, self.folder_list[folder_index], os.listdir(os.path.join(self.root_dir, self.folder_list[folder_index]))[index])
img = read_xray(image_path, 128)
target = torch.tensor(row.cancer).float()
img = np.repeat(img, 3, axis=0)
img = torch.from_numpy(img)
if self.transform:
img = self.transform(img)
return img, target
transformed_dataset = ImageDataset(root_dir="/kaggle/input/rsna-breast-cancer-detection/train_images", df=train_df)
dataloader = DataLoader(transformed_dataset, batch_size=128, shuffle=True, num_workers=2)
Model:
model = models.resnet18(pretrained=True)
num_features = model.fc.in_features
model.fc = nn.Linear(num_features, 2)
model.to(device)
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=.001)
Train Loop:
def train(model, dataloader, loss_fn, optimizer, epochs):
for epoch in tqdm(range(epochs)):
print("Using", torch.cuda.get_device_name(0))
running_loss = 0.0
for i, data in tqdm(enumerate(dataloader, 0)):
inputs, labels = data
model = model.to(device)
labels = labels.type(torch.LongTensor)
inputs, labels = inputs.to(device), labels.to(device)
optimizer.zero_grad()
outputs = model(inputs)
print(f"Out put done{i}")
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
running_loss += loss.item()
print('[Epoch %d] loss: %.3f' % (epoch + 1, running_loss / len(dataloader)))
print('Finished Training')
train(model, dataloader, criterion, optimizer, 4)
I'm following this object detection tutorial in Pytorch for Mask RCNN: https://pytorch.org/tutorials/intermediate/torchvision_tutorial.html and at the end there are some verification image.
How can I get this prediction image? I'm working with object deteciton.
Is there some way to output images from trained model that I can see is my network is learn something?
This is my code:
import os
import numpy as np
import torch
from PIL import Image
import torchvision
from torchvision.models.detection.faster_rcnn import FastRCNNPredictor
from torchvision.models.detection.mask_rcnn import MaskRCNNPredictor
#from engine import train_one_epoch, evaluate
from vision.references.detection.engine import train_one_epoch, evaluate
import utils
import transforms as T
import matplotlib.pyplot as plt
class Moj_Dataset_ArT(object):
def __init__(self, root, transforms):
self.root = root
self.transforms = transforms
# load all image files, sorting them to
# ensure that they are aligned
self.imgs = list(sorted(os.listdir(os.path.join(root, "NEW_train"))))
self.masks = list(sorted(os.listdir(os.path.join(root, "train_masks"))))
def __getitem__(self, idx):
# load images and masks
img_path = os.path.join(self.root, "NEW_train", self.imgs[idx])
mask_path = os.path.join(self.root, "train_masks", self.masks[idx])
img = Image.open(img_path).convert("RGB")
# note that we haven't converted the mask to RGB,
# because each color corresponds to a different instance
# with 0 being background
mask = Image.open(mask_path)
mask = np.array(mask)
# instances are encoded as different colors
obj_ids = np.unique(mask)
# first id is the background, so remove it
obj_ids = obj_ids[1:]
# split the color-encoded mask into a set
# of binary masks
masks = mask == obj_ids[:, None, None]
# get bounding box coordinates for each mask
num_objs = len(obj_ids)
boxes = []
for i in range(num_objs):
pos = np.where(masks[i])
xmin = np.min(pos[1])
xmax = np.max(pos[1])
ymin = np.min(pos[0])
ymax = np.max(pos[0])
boxes.append([xmin, ymin, xmax, ymax])
boxes = torch.as_tensor(boxes, dtype=torch.float32)
# there is only one class
labels = torch.ones((num_objs,), dtype=torch.int64)
masks = torch.as_tensor(masks, dtype=torch.uint8)
image_id = torch.tensor([idx])
area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])
# suppose all instances are not crowd
iscrowd = torch.zeros((num_objs,), dtype=torch.int64)
target = {}
target["boxes"] = boxes
target["labels"] = labels
target["masks"] = masks
target["image_id"] = image_id
target["area"] = area
target["iscrowd"] = iscrowd
if self.transforms is not None:
img, target = self.transforms(img, target)
return img, target
def __len__(self):
return len(self.imgs)
def get_model_instance_segmentation(num_classes):
# load an instance segmentation model pre-trained pre-trained on COCO
model = torchvision.models.detection.maskrcnn_resnet50_fpn(pretrained=False) #bilo na True
# get number of input features for the classifier
in_features = model.roi_heads.box_predictor.cls_score.in_features
# replace the pre-trained head with a new one
model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)
# now get the number of input features for the mask classifier
in_features_mask = model.roi_heads.mask_predictor.conv5_mask.in_channels
hidden_layer = 256
# and replace the mask predictor with a new one
model.roi_heads.mask_predictor = MaskRCNNPredictor(in_features_mask,
hidden_layer,
num_classes)
return model
def get_transform(train):
transforms = []
transforms.append(T.ToTensor())
#if train:
#transforms.append(T.RandomHorizontalFlip(0.5))
return T.Compose(transforms)
def draw_loss(ml):
plt.figure(figsize=(10,5))
plt.title("Training Loss")
#plt.plot(val_losses,label="val")
plt.plot(ml,label="train")
plt.xlabel("Epochs")
plt.ylabel("Loss")
plt.legend()
plt.xlim([0,80])
plt.ylim([0, 1.2])
plt.show()
def main():
# train on the GPU or on the CPU, if a GPU is not available
device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
# our dataset has two classes only - background and person
num_classes = 2
# use our dataset and defined transformations
dataset = Moj_Dataset_ArT('Train/ArT', get_transform(train=True))
dataset_test = Moj_Dataset_ArT('Train/ArT', get_transform(train=False))
# split the dataset in train and test set
indices = torch.randperm(len(dataset)).tolist()
dataset = torch.utils.data.Subset(dataset, indices[:-500])
dataset_test = torch.utils.data.Subset(dataset_test, indices[-500:])
# define training and validation data loaders
data_loader = torch.utils.data.DataLoader(
dataset, batch_size=4, shuffle=True, num_workers=4,
collate_fn=utils.collate_fn)
data_loader_test = torch.utils.data.DataLoader(
dataset_test, batch_size=1, shuffle=False, num_workers=4,
collate_fn=utils.collate_fn)
# get the model using our helper function
model = get_model_instance_segmentation(num_classes)
# move model to the right device
model.to(device)
# construct an optimizer
params = [p for p in model.parameters() if p.requires_grad]
optimizer = torch.optim.SGD(params, lr=0.0005,
momentum=0.9, weight_decay=0.0005)
# and a learning rate scheduler
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=30,
gamma=0.1) # gamma bila na 0.5
# let's train it for 10 epochs
num_epochs = 200
PATH = 'home//Train/ArT/models/'
ml =[]
for epoch in range(num_epochs):
# train for one epoch, printing every 10 iterations
loss_value = train_one_epoch(model, optimizer, data_loader, device, epoch, print_freq=10)
ml.append(loss_value)
# update the learning rate
lr_scheduler.step()
torch.save(model, PATH)
# evaluate on the test dataset
#evaluate(model, data_loader_test, device=device)
print(ml)
draw_loss(ml)
if __name__ == "__main__":
main()
Something like this
When I run this after evalueate():
img, _ = dataset_test[20]
with torch.no_grad():
prediction = model([img.to(device)])
imag = Image.fromarray(img.mul(255).permute(1, 2, 0).byte().numpy())
imaag = Image.fromarray(prediction[0]['masks'][0, 0].mul(255).byte().cpu().numpy())
imag.show()
imaag.show()
I get this (not good image):
I also tried with Detectron2
but then I need to make cfg file and train on the other way, but I'm using this Pytorch tutorial
I'm trying to create a program that show top 3 of similar image to query image, using python.
I thought Siamese Network by Triplet Loss can be good option for what I want to do.
I wrote some codes and created model with small dataset in my pc. And I inputted one of the dataset into the program to evaluate my program. I expected that the program would show same image as what I input, but the program doesn't always do so.
For example, there are five images, A, B, C, D and E. I created a model which learned the five images with Siamese Network by Triplet Loss and saved the model. And I loaded the model and input the image B for prediction, expecting that the program return B as a result, but it returns D.
When comparison of distance between dataset and query follows model training, results are as I expected. (Input Image A and return Image A)
However, after model training completed, when I load trained model and try to predict, it doesn't return correctly.
Did I do something wrong in model structure?
Or Siamese Network is not appropriate for my problem?
If structure of my code is not so bad, I guess it should be an issue of quality of the dataset.
My program is as below.
import tensorflow as tf
from tensorflow.keras import layers
from tensorflow.keras.models import Model, Sequential
import tensorflow.keras.backend as K
from tensorflow.keras.optimizers import SGD, Adam
from tensorflow.keras.applications import resnet
from tensorflow.keras.callbacks import LearningRateScheduler, ModelCheckpoint, History
import numpy as np
import random
import os
import matplotlib.pyplot as plt
from PIL import Image
from sklearn.preprocessing import LabelEncoder
import datetime
from sklearn.metrics import euclidean_distances, roc_auc_score
import load_dataset2_1 as ld
now = datetime.datetime.now()
def create_resnet(size, channel, num_classes):
input_tensor = layers.Input((size, size, channel))
ResNet50 = resnet.ResNet50(weights="imagenet", input_tensor=input_tensor, include_top=False, pooling="avg")
embedding_model = Sequential()
embedding_model.add(layers.Flatten(input_shape=ResNet50.output_shape[1:]))
embedding_model.add(layers.Dense(256, activation="relu"))
embedding_model.add(layers.BatchNormalization())
embedding_model.add(layers.Dropout(0.5))
embedding_model.add(layers.Dense(num_classes))
embedding_model = Model(inputs=ResNet50.input, outputs=embedding_model(ResNet50.output))
embedding_model.summary()
trainable = False
for layer in ResNet50.layers:
if layer.name == "conv5_block1_out":
trainable = True
layer.trainable = trainable
return embedding_model
def create_concatenate_layer(embedding_model, size, channel):
input_anchor = layers.Input(shape=(size, size, channel))
input_positive = layers.Input(shape=(size, size, channel))
input_negative = layers.Input(shape=(size, size, channel))
embedding_anchor = embedding_model(input_anchor)
embedding_positive = embedding_model(input_positive)
embedding_negative = embedding_model(input_negative)
output = layers.concatenate([embedding_anchor, embedding_positive,embedding_negative])
net = Model([input_anchor, input_positive, input_negative], output)
return net
# Online Triplet
def triplet_loss(label, embeddings):
x1 = tf.expand_dims(embeddings, axis=0)
x2 = tf.expand_dims(embeddings, axis=1)
euclidean = tf.reduce_sum((x1-x2)**2, axis=-1)
lb1 = tf.expand_dims(label[:, 0], axis=0)
lb2 = tf.expand_dims(label[:, 0], axis=1)
equal_mat = tf.equal(lb1, lb2)
# positives
positive_ind = tf.where(equal_mat)
positive_dists = tf.gather_nd(euclidean, positive_ind)
print("positive_ind : ", positive_ind)
print("positive_dists : ", positive_dists)
# negatives
negative_ind = tf.where(tf.logical_not(equal_mat))
negative_dists = tf.gather_nd(euclidean, negative_ind)
print("negative_ind : ", positive_ind)
print("negative_dists : ", positive_dists)
# [P, N]
margin = 0.25
positives = tf.expand_dims(positive_dists, axis=1)
negatives = tf.expand_dims(negative_dists, axis=0)
triplets = tf.maximum(positives - negatives + margin, 0.0)
return tf.reduce_mean(triplets)
def create_batch(x_train, y_train, size, channel, batch_size):
x_anchors = np.zeros((batch_size, size, size, channel))
x_positives = np.zeros((batch_size, size, size, channel))
x_negatives = np.zeros((batch_size, size, size, channel))
for i in range(0, batch_size):
random_index = random.randint(0, x_train.shape[0]-1)
x_anchor = x_train[random_index]
y = y_train[random_index]
dogs_for_pos = np.squeeze(np.where(y_train==y))
dogs_for_neg = np.squeeze(np.where(y_train!=y))
# print("len(dogs_for_pos) : ", len(dogs_for_pos))
# print("len(dogs_for_neg) : ", len(dogs_for_neg))
x_positive = x_train[dogs_for_pos[random.randint(0, len(dogs_for_pos)-1)]]
x_negative = x_train[dogs_for_neg[random.randint(0, len(dogs_for_neg)-1)]]
x_anchors[i] = x_anchor
x_positives[i] = x_positive
x_negatives[i] = x_negative
print("x_anchors.shape___", x_anchors.shape)
print("x_positives.shape___", x_positives.shape)
print("x_negatives.shape___", x_negatives.shape)
return [x_anchors, x_positives, x_negatives]
def data_generator(x_train, y_train, size, channel, batch_size, emb_size):
while True:
x = create_batch(x_train, y_train, size, channel, batch_size)
y = np.zeros((batch_size, 3*emb_size))
yield x, y
def train_generator(X, y_label, batch_size):
while True:
indices = np.random.permutation(X.shape[0])
for i in range(len(indices)//batch_size):
current_indices = indices[i*batch_size:(i+1)*batch_size]
X_batch = X[current_indices] / 255.0
y_batch = np.zeros((batch_size, 128), np.float32)
y_batch[:, 0] = y_label[current_indices]
yield X_batch, y_batch
def step_decay(epoch):
x = 1e-3
if epoch >= 25: x /= 10.0
if epoch >= 45: x /= 10.0
return x
size = 128
channel = 3
batch_size = 64
epochs = 1000
emb = 64
def train(folder, size=size, batch_size=batch_size, channel=channel, epochs=epochs):
print("TensorFlow version: ", tf.__version__)
print("folder : ", folder)
print("size : {0}, batch_size : {1}, channel : {2}, epochs : {3}".format(size, batch_size, channel, epochs))
switch = input("Load data : On or Off: ")
if switch == "On" or switch == "ON" or switch == "on":
switch = "On"
size = ''
(x_train, y_train), (x_test, y_test), size = ld.main(switch, folder)
x_train = np.reshape(x_train, (x_train.shape[0], size, size, channel))/255.0
x_test = np.reshape(x_test, (x_test.shape[0], size, size, channel))/255.0
# print('num_files: ', num_files)
num_classes=len(os.listdir(folder))
steps_per_epoch = x_train.shape[0]//batch_size
# opt = tf.keras.optimizers.SGD(1e-3, 0.9)
opt = tf.keras.optimizers.Adam(lr=0.0001)
scheduler = LearningRateScheduler(step_decay)
checkpoint = ModelCheckpoint("./triplet_model/model_dbt_2.hdf5", monitor="loss", save_best_only=True, save_weight_only=True)
# es_cb = tf.keras.callbacks.EarlyStopping(monitor='loss', min_delta=0.0001, patience=3, mode='auto')
net = create_resnet(size, channel, 256)
net.summary()
net.compile(loss=triplet_loss, metrics=['accuracy'], optimizer=opt)
# hist = net.fit(data_generator(x_train, y_train, size, channel, batch_size, emb_size=emb),
# steps_per_epoch=steps_per_epoch, epochs=epochs, verbose=1,
# # validation_data = (x_test, y_test),
# callbacks=[checkpoint, scheduler]
# )
hist = net.fit_generator(train_generator(x_train, y_train, batch_size),
steps_per_epoch=steps_per_epoch, epochs=epochs, verbose=1,
# validation_data = train_generator(x_test, y_test, batch_size),
callbacks=[checkpoint, scheduler],
max_queue_size = 1
)
net.save("./triplet_model/new_model.h5")
net.save("./triplet_model/new_model")
net.save_weights("./triplet_model/new_weights.hdf5", save_format="h5")
x = range(epochs)
plt.title("Model accuracy")
plt.plot(x, hist.history["accuracy"], label="accuracy")
plt.plot(x, hist.history["loss"], label="loss")
plt.xlabel("Epoch")
plt.legend(loc="center left", bbox_to_anchor=(1, 0.5), borderaxespad=0, ncol=2)
name="resnet_dogs_trian_result {0} {1}h-{2}m-{3}s.jpg".format(now.strftime("%Y-%m-%d"), now.hour, now.minute, now.second)
plt.savefig(name, bbox_inches="tight")
plt.close()
def main(test_img, folder="Base_dogs_2", base_folder="Base_dogs"):
num_files = len(os.listdir(base_folder))
# model = create_resnet(size, channel, 256)
if not os.path.isfile("./triplet_model/model_dbt_2.hdf5") and not os.path.isfile("./triplet_model/new_weights.hdf5"):
train(folder=folder)
# train(folder="Base_dogs_2")
if os.path.isfile("./triplet_model/model_dbt_2.hdf5"):
print("Loading weights: 'model_dbt_2.hdf5'")
model = tf.keras.models.load_model("./triplet_model/new_model.h5", custom_objects={'triplet_loss' : triplet_loss})
model.load_weights("./triplet_model/model_dbt_2.hdf5")
# model.summary()
elif os.path.isfile("./triplet_model/new_weights.hdf5"):
print("Loading weights: 'new_weights.hdf5'")
model = tf.keras.models.load_model("./triplet_model/new_model.h5", custom_objects={'triplet_loss' : triplet_loss})
model.load_weights("./triplet_model/new_weights.hdf5")
else:
print("Cannot load weights")
X_base=[]
Y=[]
bbb=0
#db = Dogs.objects.all()
db = os.listdir(base_folder)
print("db : ", db)
for b_img in db:
bbb += 1
file_name = b_img
b_img = base_folder + "/" + b_img
print("img_path : ", b_img)
bmg = Image.open(b_img)
bmg.show()
bmg.save("{0}{1}.jpg".format(bbb, file_name))
# bmg = bmg.convert("L")
bmg = bmg.resize((size, size))
b_data = np.asarray(bmg)
X_base.append(b_data)
Y.append(b_img)
X_base = np.array(X_base)
Y = np.array(Y)
print("X_base.shape : ", X_base.shape)
print("Y.shape : ", Y.shape)
label_ec = LabelEncoder()
label = label_ec.fit_transform(Y)
X_base = X_base.astype(np.float32)
# X_base = tf.expand_dims(X_base, axis=-1)
print("X_base.shape after expand_dims : ", X_base.shape)
(x_base, y_base) = (X_base, label)
file = test_img
print("test_img : ", file)
X=[]
img = Image.open(file)
img.show()
# img = img.convert("L")
img = img.resize((size, size))
data = np.asarray(img)
X.append(data)
X = np.array(X)
X = X.astype(np.float32)
X = np.reshape(X, (X.shape[0], size, size, channel))/255.0
print("X.shape : ", X.shape)
#X = np.expand_dims(X, axis=0)
anchor_embedding = model.predict(x_base, verbose=1)
test_embedding = model.predict(X, verbose=1)
dist_matrix = np.zeros((test_embedding.shape[0], anchor_embedding.shape[0]), np.float32)
print("dist_matrix.shape : ", dist_matrix.shape)
for i in range(dist_matrix.shape[0]):
dist_matrix[i, :] = euclidean_distances(test_embedding[i, :].reshape(1, -1), anchor_embedding)[0]
print("dist_matrix : ", dist_matrix)
#distance against query image
min_dist = np.min(dist_matrix, axis=-1)
min_idx = np.argmin(dist_matrix)
print("min_dist : ", min_dist)
print("min_idx : ", min_idx)
print("base_file_name : ", db)
time = "{0} {1}h-{2}m-{3}s_{4}".format(now.strftime('%Y-%m-%d'), now.hour, now.minute, now.second, base_folder)
print("file_name : time__", time)
results_path = []
path = 'static/results_'
# print("x_base[min_idx] : ", x_base[min_idx])
pil_img_1 = np.reshape(x_base[min_idx], (size, size, channel))
print("pil_img_1.shape : ", pil_img_1.shape)
pil_img_1 = Image.fromarray(pil_img_1.astype(np.uint8))
results_path_1 = '{0}{1}_1.jpg'.format(path, time)
pil_img_1.save(results_path_1)
results_path.append(str(results_path_1))
min_dist_2 = np.min(dist_matrix[dist_matrix > min_dist], axis=-1)
print("min_dist_2 : ", min_dist_2)
print("np.squeeze(np.where(dist_matrix==min_dist_2)) : ",np.squeeze(np.where(dist_matrix==min_dist_2)))
min_idx_2 = np.squeeze(np.where(dist_matrix==min_dist_2))[1]
print("min_idx_2 : ", min_idx_2)
img_2 = np.reshape(x_base[min_idx_2], (size, size, channel))
pil_img_2 = Image.fromarray(img_2.astype(np.uint8))
results_path_2 = '{0}{1}_2.jpg'.format(path, time)
pil_img_2.save(results_path_2)
results_path.append(str(results_path_2))
min_dist_3 = np.min(dist_matrix[dist_matrix > min_dist_2], axis=-1)
print("min_dist_3___", min_dist_3)
min_idx_3 = np.squeeze(np.where(dist_matrix==min_dist_3))[1]
print("np.squeeze(np.where(dist_matrix==min_dist_3)) : ",np.squeeze(np.where(dist_matrix==min_dist_3)))
print("min_idx_3___", min_idx_3)
img_3 = np.reshape(x_base[min_idx_3], (size, size, channel))
pil_img_3 = Image.fromarray(img_3.astype(np.uint8))
results_path_3 = '{0}{1}_3.jpg'.format(path, time)
pil_img_3.save(results_path_3)
results_path.append(str(results_path_3))
print(results_path)
dist_matrix = np.empty(0)
print("dist_matrix after clear : ", dist_matrix)
return results_path # return top 3 similar images
if __name__ == "__main__":
t = "A.jpg"
b_fn = "test"
result = main(test_img=t)
print(result)
I'm training an MRI super-resolution model on about 40k volumes with Tensorflow/Keras and a custom training loop:
Pseudo python/tf code :
for e in epochs:
for s in the steps:
batch = data_handler.get_batch(files[s])
model.train_on_batch(batch['lr'], batch['hr'])
def get_batch(volume_path) # online data augmentation and image degradation
vol_hr = np.load(volume_path) # 200x200x200 array
vol_hr = augment(vol_hr) # flips / rotation
vol_lr = degrade(vol_hr) # blur, add noise, downsample (100x100x200)
batch = crop_patches(vol_lr, vol_hr)
# batch['lr'].shape == batch_size, 32, 32, 3
# batch['hr'].shape == batch_size, 64, 64, 3
return batch
This runs terribly slow so I'm looking into optimizing the training through (i) implementing tf data pipeline (tf.data.Dataset) and (ii) Distributed training. This post is about optimizing the data pipeline.
My plan is to implement a tf data input pipeline (tf 2.4.1). The generator looks something like this at the moment:
import tensorflow as tf
import numpy as np
import os
import time
from skimage.transform import downscale_local_mean
def make_rand_vols(n, size=(200, 200, 200), tmp_path='./tmp/tmp_vol'):
if not os.path.isdir(tmp_path):
os.makedirs(tmp_path)
rand_arr_fnames = []
for f in range(n):
rand_arr = np.random.rand(size[0], size[1], size[2])
fpath = os.path.join(tmp_path, f"{f}.npy")
np.save(fpath, rand_arr)
rand_arr_fnames.append(fpath)
return rand_arr_fnames
def benchmark(dataset, num_epochs=2, steps=10):
start_time = time.perf_counter()
for epoch_num in range(num_epochs):
for s, sample in enumerate(dataset):
# Performing a training step
print(s)
time.sleep(0.01)
if s == steps - 1:
break
print("Execution time:", time.perf_counter() - start_time)
class CustomGenerator(tf.keras.utils.Sequence):
def __init__(self, fnames, batch_size=3,
scale=2, shuffle=True, patch_size=32, n_channels=3):
self.fnames = fnames
self.shuffle = shuffle
self.batch_size = batch_size
self.on_epoch_end()
self.scale = scale
self.lr_patch_size = patch_size
self.hr_patch_size = patch_size * scale
self.n_channels = n_channels
def __len__(self):
return len(self.fnames)
def on_epoch_end(self):
#'Updates indexes after each epoch'
if self.shuffle == True:
np.random.shuffle(self.fnames)
def __getitem__(self, idx):
vol_hr = np.load(self.fnames[idx]) # array of size e.g. (300, 300, 300)
vol_hr = self.augment(vol_hr)
vol_lr = self.degrade(vol_hr, self.scale) # array of size e.g. (150, 150, 300)
batch = self.get_vol_crops(vol_lr, vol_hr) # array of size (batch_size, width / scale, length / scale, depth)
# batch -> (batch_lr, batch_hr) -> (arr [batch_size, patch_lr, patch_lr, n_channels], arr [batch_size, patch_hr, patch_hr, n_channels])
return batch
def augment(self, vol):
if np.random.randint(0, 2):
vol = vol[:, ::-1, :] # simple for illustration
return vol
def degrade(self, vol_hr, downscale_factor):
vol_lr = downscale_local_mean(vol_hr,
(downscale_factor, downscale_factor, 1),
cval=0)
return vol_lr
def get_vol_crops(self, vol_lr, vol_hr):
pad = 2
rows_lr, cols_lr, n_slices = vol_lr.shape
lr_y_list = np.random.randint(0, rows_lr - self.lr_patch_size - 1, batch_size) #top left row lr
lr_x_list = np.random.randint(0, cols_lr - self.lr_patch_size - 1, batch_size) # top left col lr
z_list = np.random.randint(pad, n_slices - pad, self.batch_size)
lr_patches, hr_patches = [], []
for y_lr, x_lr, z in zip(lr_y_list, lr_x_list, z_list):
slice_idxs = [z for z in range(z - n_channels // 2, z + n_channels//2 + 1)]
# LR patch
patch_lr = vol_lr[y_lr: y_lr + self.lr_patch_size,
x_lr: x_lr + self.lr_patch_size,
slice_idxs]
print()
lr_patches.append(patch_lr)
# HR patch
y_hr, x_hr = scale * y_lr, scale * x_lr
patch_hr = vol_hr[y_hr: y_hr + self.hr_patch_size,
x_hr: x_hr + self.hr_patch_size,
slice_idxs]
#print(patch_lr.shape, patch_hr.shape)
#preview_patches(patch_lr, patch_hr)
hr_patches.append(patch_hr)
return np.array(lr_patches), np.array(hr_patches)
test_vol_fnames = make_rand_vols(50)
# Make tf.data.Dataset from keras Generator
batch_size = 24
patch_size = 32
scale = 2
n_channels = 3
gen = CustomGenerator(test_vol_fnames, patch_size=patch_size, batch_size=batch_size)
benchmark(gen, steps=50)
# Execution time: 9.38716044393368
dataset = tf.data.Dataset.from_generator(
lambda: gen,
output_types=(tf.float32, tf.float32),
output_shapes = ([batch_size, patch_size, patch_size, n_channels],
[batch_size, patch_size*scale, patch_size*scale, n_channels])
)
benchmark(dataset, steps=50)
# Execution time: 9.54540992085822
benchmark(dataset.prefetch(tf.data.AUTOTUNE), steps=50)
# Execution time: 8.615464322036132
benchmark(
tf.data.Dataset.range(2)
.interleave(
lambda _: dataset,
num_parallel_calls=tf.data.AUTOTUNE),
steps=50
)
# Execution time: 5.204080824041739
def mapped_generator(g):
return tf.py_function(lambda f: g,
inp=[f],
Tout=[tf.float32, tf.float32])
dataset_parallel = tf.data.Dataset.from_generator(
lambda: mapped_generator(gen),
output_types=(tf.float32, tf.float32),
output_shapes=([batch_size, patch_size, patch_size, n_channels],
[batch_size, patch_size*scale, patch_size*scale, n_channels])
)
benchmark(
dataset_parallel,
steps=50
)
# NameError: name 'f' is not defined
Is there anything I can do to make this more efficient? How can I wrap the generator inside tf.py_function?
I trained a network to perform semantic segmentation with dropout, and it is my understanding that as you vary the dropout keep_prob value, the output prediction changes. However, after saving the model using the tensorflow-serving method, loading it using tf.saved_model.loader.load, and varying the dropout value, I get the same output prediction value (dice score).
I followed the suggestions in this SO post, but I still get the same prediction results even if I enter 0.0.
Didn't know if it was a tensorflow issue or a bug in my code, so I tried downgrading from v1.15 to v1.10 to see if it was the former and still got the same results. I am sure it is a bug in my code now, but I am not sure where it is. A minimum working example is shown below. Could someone help me? Thank you!
This is a snippet of my training script:
#===============
def run_iteration(self, feed_dict, op_list, summaries):
output_args = self.sess.run(op_list, feed_dict=feed_dict)
return output_args
#===============
def run_epoch_train(self, curr_epoch):
print('Training over all batches')
num_total_batches = self.num_total_batches_train
curr_batch_counter = 0
# for each batch in training images
for batch in self.batch_iterator_train:
# dropout is included
if self.dropout_training_Flag == 1:
_, loss, dice = self.run_iteration(
feed_dict={
self.placeholders['images']: batch['images'],
self.placeholders['labels']: batch['labels'],
self.placeholders['is_training']: True,
self.placeholders['dropout_prob']: self.dropout_prob_training,
},
op_list=[
self.fitting_op,
self.losses[self.active_loss],
#self.outputs['sigmoid'],
self.outputs['dice'],
],
summaries=[],
)
curr_batch_counter = curr_batch_counter + 1
if (self.iteration % 5) == 0:
print('Saving model in training session')
self.saver.save(curr_epoch + 1)
This is a snippet of my testing script:
#===============
path_to_model = self.root_path_to_models + '/' + '25'
print(path_to_model)
model = tf.saved_model.loader.load( #tf.saved_model.loader.load(
sess,
[tf.saved_model.tag_constants.SERVING],
path_to_model
)
inputImage_name = model.signature_def['prediction'].inputs['images'].name
x_inp = tf.get_default_graph().get_tensor_by_name(inputImage_name)
isTraining_name = model.signature_def['prediction'].inputs['is_training'].name
tflag_op = tf.get_default_graph().get_tensor_by_name(isTraining_name)
outputs_name = model.signature_def['prediction'].outputs['sigmoid'].name
y_op = tf.get_default_graph().get_tensor_by_name(outputs_name)
if self.dropout_training_Flag == 1:
dropoutProb_name = model.signature_def['prediction'].inputs['dropout_prob'].name
dropout_prob_op = tf.get_default_graph().get_tensor_by_name(dropoutProb_name)
print(dropout_prob_op)
# iterate over batches of images
# iterate over motion category
for moCat in self.motion_categories:
# get datasets in motion category
datasets_in_moCat = d_ffn_images_labels[moCat]
dataset_name = list(datasets_in_moCat.keys())[-1]
#print(dataset_name)
loss_for_each_image = []
final_vol = np.zeros((self.original_input_image_width, self.original_input_image_height, self.num_vol_slices), dtype = np.uint8)
# get images
curr_dataset_images = datasets_in_moCat[dataset_name][0][0]
# get labels
curr_dataset_labels = datasets_in_moCat[dataset_name][0][1]
#current dataset label numbers
curr_dataset_label_numbers = d_bfnumber_images_labels[moCat][dataset_name]
#print('curr_dataset_label_numbers',curr_dataset_label_numbers)
# number of images/labels in current dataset, for current category
num_images = len(curr_dataset_images)
num_labels = len(curr_dataset_labels)
# check if num-images/labels are the same
assert(num_images == num_labels)
# load each image
for elem_idx in range(num_images):
img_path = curr_dataset_images[elem_idx]
lab_path = curr_dataset_labels[elem_idx]
xn = nib.load(img_path)
x = np.array(xn.dataobj)
labn = nib.load(lab_path)
lab = np.array(labn.dataobj)
data_affine_tform = xn.affine
# resize
xr = cv2.resize(x, (self.network_input_image_width, self.network_input_image_height), interpolation = cv2.INTER_LANCZOS4)
# standardize
y = standardize_zeroMeanUnitVar_image(copy.deepcopy(xr), self.network_input_image_width, self.network_input_image_height, self.network_input_channels)
#y = cv2.normalize(copy.deepcopy(xr), None, alpha=0, beta=1, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_32F)
# match network input -- [height, width, channels]
y = np.reshape(y, newshape=(self.network_input_image_height, self.network_input_image_width, self.network_input_channels))
# append to match network input -- [batch, height, width, channels]
input_list = []
input_list.append(y)
input_list = np.asarray(input_list).astype(np.float32)
# ======================
# MODIFY DROPOUT HERE FROM JSON FILE
# CHANGED VALUES FROM 0.0, 0.5, 1.0 -- same prediction score
# ======================
# run and get output
if self.dropout_training_Flag == 1:
output = sess.run(y_op, feed_dict={x_inp: input_list, tflag_op: True, dropout_prob_op: self.dropout_prob_testing})
else:
output = sess.run(y_op, feed_dict={x_inp: input_list, tflag_op: False})
tmpOut = cv2.resize(output[0,:,:,0], (self.original_input_image_width, self.original_input_image_height), interpolation = cv2.INTER_LANCZOS4)
prediction = np.asarray((tmpOut > 0.5))
labels = np.asarray((lab > 0))
EPS = 0.0000001
#output_original = cv2.resize(output[0,:,:,0], (original_input_image_width, original_input_image_height), interpolation = cv2.INTER_LANCZOS4)
loss = 2.0 * np.sum(labels * prediction, axis=(0, 1)) / (np.sum(labels ** 2 + prediction ** 2, axis=(0, 1)) + EPS)
loss_for_each_image.append(loss)
#place slice in final_vol
#print(curr_dataset_label_numbers[elem_idx][1])
#print(type(curr_dataset_label_numbers[elem_idx][1]))
final_vol[:,:,curr_dataset_label_numbers[elem_idx][1] - 1] = np.asarray(prediction*255.0).astype(np.uint8)
# dice mean over dataset
dice_mean_for_dataset = np.mean(loss_for_each_image)
print(dataset_name, dice_mean_for_dataset)
self.diceScore_for_each_dataset.append(dice_mean_for_dataset)
self.list_dataset_name.append(dataset_name)
This is the code for the inputs/outputs:
#===============
def create_placeholders(self):
self.placeholders['images'] = tf.placeholder(
shape=[None] + self.network_input_size + [self.network_input_channels],
name='images',
dtype=tf.float32
)
self.placeholders['labels'] = tf.placeholder(
shape=[None] + self.network_input_size + [self.network_output_channels],
name='labels',
dtype=tf.float32
)
self.placeholders['is_training'] = tf.placeholder(
shape=[],
name='is_training',
dtype=tf.bool
)
# dropout is included
if self.dropout_training_Flag == 1:
self.placeholders['dropout_prob'] = tf.placeholder(
shape=[],
name='dropout_prob',
dtype=tf.float32
)
#===============
def create_outputs(self):
if self.network_name == 'UNet':
print('\n')
print('Training UNet')
# dropout is included
if self.dropout_training_Flag == 1:
# train with dropout
unet_output = unet_dropout(
self.placeholders['images'],
self.placeholders['is_training'],
self.placeholders['dropout_prob'],
self.network_output_channels
)
if self.network_output_channels == 1:
self.outputs['sigmoid'] = unet_output
else:
self.outputs['sigmoid'] = unet_output
This is the code for my model:
#===============
def batch_norm_relu(inputs, is_training):
net = slim.batch_norm(inputs, is_training=is_training)
net = tf.nn.relu(net)
return net
#===============
def dropout (input, keep_prob, is_training):
if is_training == True:
dropout = tf.nn.dropout(input, keep_prob)
else:
dropout = input
return dropout
#===============
def model(inputs, is_training, keep_prob, num_classes):
with tf.variable_scope("model", reuse=tf.AUTO_REUSE):
base_num_kernels = 64
# =================================
# encoder
# 256
x = conv2d_fixed_padding(inputs=inputs, filters=base_num_kernels, kernel_size=3, stride=1)
x = batch_norm_relu(x, is_training)
x = conv2d_fixed_padding(inputs=x, filters=base_num_kernels, kernel_size=3, stride=1)
x = batch_norm_relu(x, is_training)
output_b1 = x
output_list_b1 = [x]
output_b1 = dropout(output_b1, keep_prob, is_training)
output_b1 = tf.layers.max_pooling2d(inputs=output_b1, pool_size=2, strides=2, padding='SAME')
# =================================
# 128
x = conv2d_fixed_padding(inputs=output_b1, filters=2*base_num_kernels, kernel_size=3, stride=1)
x = batch_norm_relu(x, is_training)
x = conv2d_fixed_padding(inputs=x, filters=2*base_num_kernels, kernel_size=3, stride=1)
x = batch_norm_relu(x, is_training)
output_b2 = x
output_list_b2 = [x]
output_b2 = dropout(output_b2, keep_prob, is_training)
# =================================
# decoder
# 128 -> 256
output_b3 = conv2d_transpose(output_b2, kernel_size=2, output_channels=base_num_kernels)
output_b4 = tf.concat([output_b3, x], axis=3)
# =================================
# 256
conv_final = conv2d_fixed_padding(inputs=output_b4, filters=base_num_kernels, kernel_size=3, stride=1)
conv_final = batch_norm_relu(conv_final, is_training)
conv_final = conv2d_fixed_padding(inputs=conv_final, filters=base_num_kernels, kernel_size=3, stride=1)
conv_final = batch_norm_relu(conv_final, is_training)
# =================================
# output
outputs = conv2d_fixed_padding(inputs=conv_final, filters=num_classes, kernel_size=3, stride=1)
if num_classes == 1:
outputs = tf.nn.sigmoid(outputs)
else:
h = outputs.get_shape().as_list()[1]
w = outputs.get_shape().as_list()[2]
outputs_reshaped = tf.reshape(outputs, np.asarray([-1, num_classes]))
outputs_final = tf.nn.softmax(outputs_reshaped)
outputs = tf.reshape(outputs_final, np.asarray([-1, h, w, num_classes]))
return outputs
This is the way that I save the network weights:
#===============
def __create_summary_manager(self):
self.saver = Saver(
self.sess,
self.placeholders,
self.outputs,
self.savepath
)
#===============
import tensorflow as tf
class Saver(object):
def __init__(self, sess, input_dict, output_dict, path):
self.sess = sess
self.input_dict = input_dict
self.output_dict = output_dict
self.path = path
self.iteration = 0
self.input_dict_info = {}
self.output_dict_info = {}
for key in input_dict.keys():
self.input_dict_info[key] = \
tf.saved_model.utils.build_tensor_info(
self.input_dict[key]
)
for key in output_dict.keys():
self.output_dict_info[key] = \
tf.saved_model.utils.build_tensor_info(
self.output_dict[key]
)
self.prediction_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs=self.input_dict_info,
outputs=self.output_dict_info)
)
def save(self, iteration_val):
self.iteration += 1
export_path = os.path.join(
tf.compat.as_bytes(self.path),
tf.compat.as_bytes(str(iteration_val))
)
self.builder = tf.saved_model.builder.SavedModelBuilder(export_path)
self.builder.add_meta_graph_and_variables(
self.sess, [tf.saved_model.tag_constants.SERVING],
signature_def_map={
'prediction': self.prediction_signature,
}
)
self.builder.save()