Related
I'm currently using CoAtNet0 for this project, and I can't seem to save the model. Hope someone can guide me how to fix the error or is there another way to save the model? The error for the code is:
AssertionError: Tried to export a function which references
'untracked' resource Tensor("272554:0", shape=(), dtype=resource).
TensorFlow objects (e.g. tf.Variable) captured by functions must be
'tracked' by assigning them to an attribute of a tracked object or
assigned to an attribute of the main object directly.
Here's the code for the model.
# CoAtNet
class MBConv(tf.keras.layers.Layer):
def __init__(self, filters, kernel_size, strides = 1, expand_ratio = 1, se_ratio = 4, residual = True, momentum = 0.9, epsilon = 0.01, convolution = tf.keras.layers.Conv2D, activation = tf.nn.swish, kernel_initializer = "he_normal", **kwargs):
super(MBConv, self).__init__(**kwargs)
self.filters = filters
self.kernel_size = kernel_size
self.strides = strides
self.expand_ratio = expand_ratio
self.se_ratio = se_ratio
self.residual = residual
self.momentum = momentum
self.epsilon = epsilon
self.convolution = convolution
self.activation = activation
self.kernel_initializer = kernel_initializer
self.model_layer = layers.LayerNormalization()
def build(self, input_shape):
self.layers = []
self.post = []
if self.expand_ratio != 1:
conv = self.convolution(input_shape[-1] * self.expand_ratio, 1, use_bias = False, kernel_initializer = self.kernel_initializer)
norm = tf.keras.layers.BatchNormalization(momentum = self.momentum, epsilon = self.epsilon)
act = tf.keras.layers.Activation(self.activation)
input_shape = input_shape[:-1] + (input_shape[-1] * self.expand_ratio,)
self.layers += [conv, norm, act]
#Depthwise Convolution
conv = self.convolution(input_shape[-1], self.kernel_size, strides = self.strides, groups = input_shape[-1], padding = "same", use_bias = False, kernel_initializer = self.kernel_initializer)
norm = tf.keras.layers.BatchNormalization(momentum = self.momentum, epsilon = self.epsilon)
act = tf.keras.layers.Activation(self.activation)
self.layers += [conv, norm, act]
#Squeeze and Excitation layer, if desired
axis = list(range(1, len(input_shape) - 1))
gap = tf.keras.layers.Lambda(lambda x: tf.reduce_mean(x, axis = axis, keepdims = True))
squeeze = self.convolution(max(1, int(input_shape[-1] / self.se_ratio)), 1, use_bias = True, kernel_initializer = self.kernel_initializer)
act = tf.keras.layers.Activation(self.activation)
excitation = self.convolution(input_shape[-1], 1, use_bias = True, kernel_initializer = self.kernel_initializer)
se = lambda x: x * tf.nn.sigmoid(excitation(act(squeeze(gap(x)))))
self.layers += [se]
#Output Phase
conv = self.convolution(self.filters, 1, use_bias = False, kernel_initializer = self.kernel_initializer)
norm = tf.keras.layers.BatchNormalization(momentum = self.momentum, epsilon = self.epsilon)
self.layers += [conv, norm]
#Residual
if self.residual:
if 1 < self.strides:
pool = tf.keras.layers.MaxPool2D(pool_size = self.strides + 1, strides = self.strides, padding = "same")
self.post.append(pool)
if input_shape[-1] != self.filters:
resample = self.convolution(self.filters, 1, use_bias = False, kernel_initializer = self.kernel_initializer)
self.post.append(resample)
def call(self, x):
out = x
for layer in self.layers:
out = layer(out)
if self.residual:
for layer in self.post:
x = layer(x)
out = out + x
return out
def get_config(self):
config = super(MBConv, self).get_config()
config["filters"] = self.filters
config["kernel_size"] = self.kernel_size
config["expand_ratio"] = self.expand_ratio
config["se_ratio"] = self.se_ratio
config["residual"] = self.residual
config["momentum"] = self.momentum
config["epsilon"] = self.epsilon
config["convolution"] = self.convolution
config["activation"] = self.activation
config["kernel_initializer"] = self.kernel_initializer
return config
class MultiHeadSelfAttention(tf.keras.layers.Layer):
def __init__(self, emb_dim = 768, n_head = 12, out_dim = None, relative_window_size = None, dropout_rate = 0., kernel_initializer = tf.keras.initializers.RandomNormal(mean = 0, stddev = 0.01), **kwargs):
#ScaledDotProductAttention
super(MultiHeadSelfAttention, self).__init__(**kwargs)
self.emb_dim = emb_dim
self.n_head = n_head
if emb_dim % n_head != 0:
raise ValueError("Shoud be embedding dimension % number of heads = 0.")
if out_dim is None:
out_dim = self.emb_dim
self.out_dim = out_dim
if relative_window_size is not None and np.ndim(relative_window_size) == 0:
relative_window_size = [relative_window_size, relative_window_size]
self.relative_window_size = relative_window_size
self.projection_dim = emb_dim // n_head
self.dropout_rate = dropout_rate
self.query = tf.keras.layers.Dense(emb_dim, kernel_initializer = kernel_initializer)
self.key = tf.keras.layers.Dense(emb_dim, kernel_initializer = kernel_initializer)
self.value = tf.keras.layers.Dense(emb_dim, kernel_initializer = kernel_initializer)
self.combine = tf.keras.layers.Dense(out_dim, kernel_initializer = kernel_initializer)
def build(self, input_shape):
if self.relative_window_size is not None:
self.relative_position_bias_table = self.add_weight("relative_position_bias_table", shape = [((2 * self.relative_window_size[0]) - 1) * ((2 * self.relative_window_size[1]) - 1), self.n_head], trainable = self.trainable)
coords_h = np.arange(self.relative_window_size[0])
coords_w = np.arange(self.relative_window_size[1])
coords = np.stack(np.meshgrid(coords_h, coords_w, indexing = "ij")) #2, Wh, Ww
coords = np.reshape(coords, [2, -1])
relative_coords = np.expand_dims(coords, axis = -1) - np.expand_dims(coords, axis = -2) #2, Wh * Ww, Wh * Ww
relative_coords = np.transpose(relative_coords, [1, 2, 0]) #Wh * Ww, Wh * Ww, 2
relative_coords[:, :, 0] += self.relative_window_size[0] - 1 #shift to start from 0
relative_coords[:, :, 1] += self.relative_window_size[1] - 1
relative_coords[:, :, 0] *= 2 * self.relative_window_size[1] - 1
relative_position_index = np.sum(relative_coords, -1)
self.relative_position_index = tf.Variable(tf.convert_to_tensor(relative_position_index), trainable = False, name= "relative_position_index")
def attention(self, query, key, value, relative_position_bias = None):
score = tf.matmul(query, key, transpose_b = True)
n_key = tf.cast(tf.shape(key)[-1], tf.float32)
scaled_score = score / tf.math.sqrt(n_key)
if relative_position_bias is not None:
scaled_score = scaled_score + relative_position_bias
weight = tf.nn.softmax(scaled_score, axis = -1)
if 0 < self.dropout_rate:
weight = tf.nn.dropout(weight, self.dropout_rate)
out = tf.matmul(weight, value)
return out
def separate_head(self, x):
out = tf.keras.layers.Reshape([-1, self.n_head, self.projection_dim])(x)
out = tf.keras.layers.Permute([2, 1, 3])(out)
return out
def call(self, inputs):
query = self.query(inputs)
key = self.key(inputs)
value = self.value(inputs)
query = self.separate_head(query)
key = self.separate_head(key)
value = self.separate_head(value)
relative_position_bias = None
if self.relative_window_size is not None:
relative_position_bias = tf.gather(self.relative_position_bias_table, tf.reshape(self.relative_position_index, [-1]))
relative_position_bias = tf.reshape(relative_position_bias, [self.relative_window_size[0] * self.relative_window_size[1], self.relative_window_size[0] * self.relative_window_size[1], -1]) #Wh * Ww,Wh * Ww, nH
relative_position_bias = tf.transpose(relative_position_bias, [2, 0, 1]) #nH, Wh * Ww, Wh * Ww
relative_position_bias = tf.expand_dims(relative_position_bias, axis = 0)
attention = self.attention(query, key, value, relative_position_bias)
attention = tf.keras.layers.Permute([2, 1, 3])(attention)
attention = tf.keras.layers.Reshape([-1, self.emb_dim])(attention)
out = self.combine(attention)
return out
def get_config(self):
config = super(MultiHeadSelfAttention, self).get_config()
config["emb_dim"] = self.emb_dim
config["n_head"] = self.n_head
config["out_dim"] = self.out_dim
config["relative_window_size"] = self.relative_window_size
config["projection_dim"] = self.projection_dim
config["dropout_rate"] = self.dropout_rate
return config
class ConvTransformer(tf.keras.layers.Layer):
def __init__(self, emb_dim = 768, n_head = 12, strides = 1, out_dim = None, epsilon = 1e-5, dropout_rate = 0., activation = tf.keras.activations.gelu, kernel_initializer = tf.keras.initializers.RandomNormal(mean = 0, stddev = 0.01), **kwargs):
super(ConvTransformer, self).__init__(**kwargs)
self.emb_dim = emb_dim
self.n_head = n_head
self.strides = strides
self.out_dim = out_dim if out_dim is not None else emb_dim
self.epsilon = epsilon
self.dropout_rate = dropout_rate
self.activation = activation
self.kernel_initializer = kernel_initializer
def build(self, input_shape):
self.attention = []
self.residual = []
#Attention
shape = input_shape[1:3]
if 1 < self.strides:
shape = np.divide(np.add(shape, (self.strides - 1)), self.strides).astype(int)
pool = tf.keras.layers.MaxPool2D(pool_size = self.strides + 1, strides = self.strides, padding = "same")
self.attention.append(pool)
self.residual.append(pool)
if input_shape[-1] != self.out_dim:
resample = tf.keras.layers.Conv2D(self.out_dim, 1, padding = "same", use_bias = False, kernel_initializer = "he_normal")
self.residual.append(resample)
pre_reshape = tf.keras.layers.Reshape([-1, input_shape[-1]])
mhsa = MultiHeadSelfAttention(emb_dim = self.emb_dim, n_head = self.n_head, out_dim = self.out_dim, relative_window_size = shape, dropout_rate = self.dropout_rate)
post_reshape = tf.keras.layers.Reshape([*shape, self.out_dim])
self.attention += [pre_reshape, mhsa, post_reshape]
self.ffn = []
#Feed Forward Network
norm = tf.keras.layers.LayerNormalization(epsilon = self.epsilon)
dense1 = tf.keras.layers.Dense(self.out_dim, kernel_initializer = self.kernel_initializer)
act = tf.keras.layers.Activation(self.activation)
dense2 = tf.keras.layers.Dense(self.out_dim, kernel_initializer = self.kernel_initializer)
self.ffn = [norm, dense1, act, dense2]
def call(self, inputs):
out = inputs
for layer in self.attention:
out = layer(out)
for layer in self.residual:
inputs = layer(inputs)
out = out + inputs
for layer in self.ffn:
out = layer(out)
return out
def get_config(self):
config = super(ConvTransformer, self).get_config()
config["emb_dim"] = self.emb_dim
config["n_head"] = self.n_head
config["strides"] = self.strides
config["out_dim"] = self.out_dim
config["epsilon"] = self.epsilon
config["dropout_rate"] = self.dropout_rate
config["activation"] = self.activation
config["kernel_initializer"] = self.kernel_initializer
return config
def coatnet(x, n_class = 1000, include_top = True, n_depth = [2, 2, 6, 14, 2], n_feature = [64, 96, 192, 384, 768], block = ["C", "M", "M", "T", "T"], stage_stride_size = 2, expand_ratio = 4, se_ratio = 4, dropout_rate = 0., activation = tf.keras.activations.gelu, name = ""):
#block : S > Stem, C > MBConv, T > Transformer
if 0 < len(name):
name += "_"
if isinstance(stage_stride_size, int):
stage_stride_size = [stage_stride_size] * len(block)
out = x
for i, (_n_depth, _n_feature, _block, _stage_stride_size) in enumerate(zip(n_depth, n_feature, block, stage_stride_size)):
for j in range(_n_depth):
stride_size = 1 if j != 0 else _stage_stride_size
residual = out
if _block.upper() == "C":# i == 0:
out = tf.keras.layers.Conv2D(_n_feature, 1 if i != 0 else 3, strides = stride_size, padding = "same", use_bias = False, kernel_initializer = "he_normal", name = "{0}stage{1}_conv{2}".format(name, i, j + 1))(out)
out = tf.keras.layers.BatchNormalization(momentum = 0.9, epsilon = 1e-5, name = "{0}stage{1}_norm{2}".format(name, i, j + 1))(out)
out = tf.keras.layers.Activation(activation, name = "{0}stage{1}_act{2}".format(name, i, j + 1))(out)
elif _block.upper() == "M":
out = tf.keras.layers.BatchNormalization(momentum = 0.9, epsilon = 1e-5, name = "{0}stage{1}_pre_norm{2}".format(name, i, j + 1))(out)
out = MBConv(_n_feature, 3, strides = stride_size, expand_ratio = expand_ratio, se_ratio = se_ratio, residual = True, momentum = 0.9, epsilon = 1e-5, activation = activation, name = "{0}stage{1}_mbconv{2}".format(name, i, j + 1))(out)
elif _block.upper() == "T":
out = tf.keras.layers.LayerNormalization(epsilon = 1e-5, name = "{0}stage{1}_pre_norm{2}".format(name, i, j + 1))(out)
out = ConvTransformer(32 * 8, 8, strides = stride_size, out_dim = _n_feature, epsilon = 1e-5, activation = activation, name = "{0}stage{1}_transformer{2}".format(name, i, j + 1))(out)
if include_top:
out = tf.keras.layers.GlobalAveragePooling2D(name = "{0}gap".format(name))(out)
if 0 < dropout_rate:
out = tf.keras.layers.Dropout(dropout_rate, name = "{0}dropout".format(name))(out)
out = tf.keras.layers.Dense(n_class, kernel_initializer = tf.keras.initializers.RandomNormal(mean = 0, stddev = 0.01), name = "{0}logits".format(name))(out)
return out
def coatnet0(input_tensor = None, input_shape = None, classes = 1000, include_top = True, weights = None):
if input_tensor is None:
img_input = tf.keras.layers.Input(shape = input_shape)
else:
if not tf.keras.backend.is_keras_tensor(input_tensor):
img_input = tf.keras.layers.Input(tensor = input_tensor, shape = input_shape)
else:
img_input = input_tensor
out = coatnet(img_input, classes, include_top, n_depth = [2, 2, 3, 5, 2], n_feature = [64, 96, 192, 384, 768], block = ["C", "M", "M", "T", "T"], stage_stride_size = 2, expand_ratio = 4, se_ratio = 4, dropout_rate = 0., activation = tf.keras.activations.gelu)
model = tf.keras.Model(img_input, out)
if weights is not None:
model.load_weights(weights)
return model
def get_model():
model = coatnet0(input_shape = (224, 224, 3), include_top = False)
for layer in model.layers[:-1]:
layer.trainable = False
#adding layers
x = tf.keras.layers.Flatten()(model.output)
#x = tf.keras.layers.BatchNormalization()(x)
#x = tf.keras.layers.Dense(500, activation = tf.keras.activations.gelu)(x)
x = tf.keras.layers.Dense(500, activation = tf.keras.activations.gelu, kernel_initializer=tf.keras.initializers.VarianceScaling()`)(x)`
#x = tf.keras.layers.Dropout(0.2)(x)
#x = tf.keras.layers.Dense(500, activation = tf.keras.activations.gelu)(x)
x = tf.keras.layers.Dense(500, activation = tf.keras.activations.gelu, kernel_initializer=tf.keras.initializers.VarianceScaling()
)(x)
prediction = tf.keras.layers.Dense(2, activation = 'softmax', kernel_initializer=tf.keras.initializers.VarianceScaling()
)(x)
model = tf.keras.Model(model.input, prediction)
model.summary()
loss = tf.keras.losses.binary_crossentropy
opt = tf.keras.optimizers.Adam(learning_rate=0.00001)
metric = ['accuracy']
#weights = compute_class_weight(class_weight = "balanced", classes = np.unique(train_batches.classes), y = train_batches.classes)
#cw = dict(zip(np.unique(train_batches.classes), weights))
callbacks = [
#tf.keras.callbacks.ModelCheckpoint("covid_classifier_model.h1", save_best_only=True, verbose = 0),
tf.keras.callbacks.EarlyStopping(patience=10, monitor='val_loss', mode = "auto", verbose=1),
tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3, verbose=1, mode='auto')
]
model.compile(optimizer = opt, loss = loss,
metrics=metric)
return model
model.save("my_model")
I've been struggling to find what's wrong in my code. I'm trying to implement DCGAN paper and from the past 2 days, I'm going through these errors. Could anyone please help me fix this?
I'm training this on Google colab with GPU runtime but I'm getting this error. Yesterday, I implemented the first GAN paper by Ian Goodfellow and I did not got this error. I don't know what's happening any help would be appreciated. Also, please check whether the gen_input is correct or not.
I already asked this question and no one replied to the old post. one person replied to it but all he said was to change the lines up and down which also gives the same error. please please help me
Here is the code:
import torch
import numpy as np
import torch.nn as nn
import torch.nn.functional as F
import torchvision
import torchvision.transforms as transforms
from torchvision.utils import save_image
import torch.optim as optim
lr = 0.00002 #learning rate
nc = 3 #color channels
nz = 100 #size of latent vector or size of generator input
ngf = 64 #size of feature maps in generator
ndf = 64 #size of feature maps in discriminator
height = 128 #height of the image
width = 128 #width of the image
num_epochs = 5 #the variable name tells everything
workers = 2 #number of workers to load the data in batches
batch_size = 64 #batch size
image_size = 128 #resizing parameter
root = './simpsons/' #path to the training directory
beta1 = 0.5
img_shape = (nc, height, width)
class Generator(nn.Module):
def __init__(self):
super(Generator, self).__init__()
self.convt1 = nn.ConvTranspose2d(in_channels = nz, out_channels = ngf*8, kernel_size = 4, stride = 1, padding = 0, bias = False)
self.convt2 = nn.ConvTranspose2d(in_channels = ngf*8, out_channels = ngf*4, kernel_size = 4, stride = 2, padding = 1, bias = False)
self.convt3 = nn.ConvTranspose2d(in_channels = ngf*4, out_channels = ngf*2, kernel_size = 4, stride = 2, padding = 1, bias = False)
self.convt4 = nn.ConvTranspose2d(in_channels = ngf*2, out_channels = ngf, kernel_size = 4, stride = 2, padding = 1, bias = False)
self.convt5 = nn.ConvTranspose2d(in_channels = ngf, out_channels = 3, kernel_size=4, stride = 2, padding = 1, bias = False)
def forward(self, t):
t = self.convt1(t)
t = nn.BatchNorm2d(t)
t = F.relu(t)
t = self.convt2(t)
t = nn.BatchNorm2d(t)
t = F.relu(t)
t = self.convt3(t)
t = nn.BatchNorm2d(t)
t = F.relu(t)
t = self.convt4(t)
t = nn.BatchNorm2d(t)
t = F.relu(t)
t = self.convt5(t)
t = F.tanh(t)
return t
class Discriminator(nn.Module):
def __init__(self):
super(Discriminator, self).__init__()
self.conv1 = nn.Conv2d(in_channels = 3, out_channels = ndf, kernel_size = 4, stride = 2, padding = 1, bias = False)
self.conv2 = nn.Conv2d(in_channels = ndf, out_channels = ndf*2, kernel_size = 4, stride = 2, padding = 1, bias = False)
self.conv3 = nn.Conv2d(in_channels = ndf*2, out_channels = ndf*4, kernel_size = 4, stride = 2, padding = 1, bias = False)
self.conv4 = nn.Conv2d(in_channels = ndf*4, out_channels = ndf*8, kernel_size = 4, stride = 2, padding = 1, bias = False)
self.conv5 = nn.Conv2d(in_channels = ndf*8, out_channels = 1, kernel_size = 4, stride = 1, padding = 0, bias = False)
def forward(self, t):
t = self.conv1(t)
t = F.leaky_relu(t, 0.2)
t = self.conv2(t)
t = nn.BatchNorm2d(t)
t = F.leaky_relu(t, 0.2)
t = self.conv3(t)
t = nn.BatchNorm2d(t)
t = F.leaky_relu(t, 0.2)
t = self.conv4(t)
t = nn.BatchNorm2d(t)
t = F.leaky_relu(t, 0.2)
t = self.conv5(t)
t = F.sigmoid(t)
return t
def weights_init(m):
classname = m.__class__.__name__ #returns the class name(eg: Conv2d or ConvTranspose2d)
if classname.find('Conv') != -1:
nn.init.normal_(m.weight.data, 0.0, 0.02) #0.0 is mean and 0.02 is standard deviation
elif classname.find('BatchNorm') != -1:
nn.init.normal_(m.weight.data, 1, 0.02) #1 is mean and 0.02 is standard deviation
nn.init.constant_(m.bias.data, 0.0)
def load_data(image_size, root):
transform = transforms.Compose([
transforms.Resize(image_size),
transforms.ToTensor(),
transforms.Normalize((0.486, 0.486, 0.486), (0.486, 0.486, 0.486))
])
train_set = torchvision.datasets.ImageFolder(root = root, transform = transform)
return train_set
#getting the batches of data
train_set = load_data(image_size, root)
dataloader = torch.utils.data.DataLoader(train_set, batch_size = batch_size, shuffle = True, num_workers = workers)
generator = Generator()
discriminator = Discriminator()
generator.apply(weights_init)
discriminator.apply(weights_init)
print(generator)
print(discriminator)
criterion = nn.BCELoss()
noise = torch.randn(64, nz, 1, 1)
optimizer_G = optim.Adam(generator.parameters(), lr = lr, betas=(beta1, 0.999))
optimizer_D = optim.Adam(discriminator.parameters(), lr = lr, betas=(beta1, 0.999))
if torch.cuda.is_available():
print("CUDA available")
generator = generator.to('cuda')
discriminator = discriminator.to('cuda')
criterion = criterion.cuda('cuda')
Tensor = torch.cuda.FloatTensor
print("Networks moved on to cuda")
for epoch in range(num_epochs):
for i, (images, labels) in enumerate(dataloader):
val = Tensor(images.size(0), 1).fill_(1.0)
fake = Tensor(images.size(0),1).fill_(0.0)
real_images = images
optimizer_G.zero_grad()
gen_input = Tensor(np.random.normal(0,1,(512,100,4,4)))
gen = generator(gen_input)
g_loss = loss_func(discriminator(gen), val)
g_loss.backward()
optimizer_G.step()
optimizer_D.zero_grad()
real_loss = loss_func(discriminator(real_images), val)
fake_loss = loss_func(discriminator(gen.detach()),fake)
d_loss = (real_loss + fake_loss)/2
d_loss.backward()
optimizer_D.step()
if i%900 == 0:
print("[EPOCH %d/%d] [Batch %d/%d] [D loss: %f] [G loss: %f]"%(epoch, num_epochs, i, len(dataset), d_loss.item(), g_loss.item()))
total_batch = epoch * len(dataset) + i
if total_batch%400 == 0:
save_image(gen.data[:25], 'output/%d.png' % total_batch, nrow=5)
And here's the error:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-36-0af32f223344> in <module>()
18 gen_input = gen_input.cuda()
19 #we then pass it to generator()
---> 20 gen = generator(gen_input) #this returns a image
21
22 #now calculate the loss wrt to discriminator output
/usr/local/lib/python3.6/dist-packages/torch/nn/modules/batchnorm.py in __init__(self, num_features, eps, momentum, affine, track_running_stats)
40 self.track_running_stats = track_running_stats
41 if self.affine:
---> 42 self.weight = Parameter(torch.Tensor(num_features))
43 self.bias = Parameter(torch.Tensor(num_features))
44 else:
TypeError: expected CPU (got CUDA)
Any help would be appreciated. Thank you!
You are using nn.BatchNorm2d in a wrong way.
BatchNorm is a layer, just like Conv2d. It has internal parameters and buffers.
Therefore, you must define these layers in the __init__ of your generator/discriminator.
Right now, you define the layer in your forward pass - this is wrong in so many ways...
I trained a network to perform semantic segmentation with dropout, and it is my understanding that as you vary the dropout keep_prob value, the output prediction changes. However, after saving the model using the tensorflow-serving method, loading it using tf.saved_model.loader.load, and varying the dropout value, I get the same output prediction value (dice score).
I followed the suggestions in this SO post, but I still get the same prediction results even if I enter 0.0.
Didn't know if it was a tensorflow issue or a bug in my code, so I tried downgrading from v1.15 to v1.10 to see if it was the former and still got the same results. I am sure it is a bug in my code now, but I am not sure where it is. A minimum working example is shown below. Could someone help me? Thank you!
This is a snippet of my training script:
#===============
def run_iteration(self, feed_dict, op_list, summaries):
output_args = self.sess.run(op_list, feed_dict=feed_dict)
return output_args
#===============
def run_epoch_train(self, curr_epoch):
print('Training over all batches')
num_total_batches = self.num_total_batches_train
curr_batch_counter = 0
# for each batch in training images
for batch in self.batch_iterator_train:
# dropout is included
if self.dropout_training_Flag == 1:
_, loss, dice = self.run_iteration(
feed_dict={
self.placeholders['images']: batch['images'],
self.placeholders['labels']: batch['labels'],
self.placeholders['is_training']: True,
self.placeholders['dropout_prob']: self.dropout_prob_training,
},
op_list=[
self.fitting_op,
self.losses[self.active_loss],
#self.outputs['sigmoid'],
self.outputs['dice'],
],
summaries=[],
)
curr_batch_counter = curr_batch_counter + 1
if (self.iteration % 5) == 0:
print('Saving model in training session')
self.saver.save(curr_epoch + 1)
This is a snippet of my testing script:
#===============
path_to_model = self.root_path_to_models + '/' + '25'
print(path_to_model)
model = tf.saved_model.loader.load( #tf.saved_model.loader.load(
sess,
[tf.saved_model.tag_constants.SERVING],
path_to_model
)
inputImage_name = model.signature_def['prediction'].inputs['images'].name
x_inp = tf.get_default_graph().get_tensor_by_name(inputImage_name)
isTraining_name = model.signature_def['prediction'].inputs['is_training'].name
tflag_op = tf.get_default_graph().get_tensor_by_name(isTraining_name)
outputs_name = model.signature_def['prediction'].outputs['sigmoid'].name
y_op = tf.get_default_graph().get_tensor_by_name(outputs_name)
if self.dropout_training_Flag == 1:
dropoutProb_name = model.signature_def['prediction'].inputs['dropout_prob'].name
dropout_prob_op = tf.get_default_graph().get_tensor_by_name(dropoutProb_name)
print(dropout_prob_op)
# iterate over batches of images
# iterate over motion category
for moCat in self.motion_categories:
# get datasets in motion category
datasets_in_moCat = d_ffn_images_labels[moCat]
dataset_name = list(datasets_in_moCat.keys())[-1]
#print(dataset_name)
loss_for_each_image = []
final_vol = np.zeros((self.original_input_image_width, self.original_input_image_height, self.num_vol_slices), dtype = np.uint8)
# get images
curr_dataset_images = datasets_in_moCat[dataset_name][0][0]
# get labels
curr_dataset_labels = datasets_in_moCat[dataset_name][0][1]
#current dataset label numbers
curr_dataset_label_numbers = d_bfnumber_images_labels[moCat][dataset_name]
#print('curr_dataset_label_numbers',curr_dataset_label_numbers)
# number of images/labels in current dataset, for current category
num_images = len(curr_dataset_images)
num_labels = len(curr_dataset_labels)
# check if num-images/labels are the same
assert(num_images == num_labels)
# load each image
for elem_idx in range(num_images):
img_path = curr_dataset_images[elem_idx]
lab_path = curr_dataset_labels[elem_idx]
xn = nib.load(img_path)
x = np.array(xn.dataobj)
labn = nib.load(lab_path)
lab = np.array(labn.dataobj)
data_affine_tform = xn.affine
# resize
xr = cv2.resize(x, (self.network_input_image_width, self.network_input_image_height), interpolation = cv2.INTER_LANCZOS4)
# standardize
y = standardize_zeroMeanUnitVar_image(copy.deepcopy(xr), self.network_input_image_width, self.network_input_image_height, self.network_input_channels)
#y = cv2.normalize(copy.deepcopy(xr), None, alpha=0, beta=1, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_32F)
# match network input -- [height, width, channels]
y = np.reshape(y, newshape=(self.network_input_image_height, self.network_input_image_width, self.network_input_channels))
# append to match network input -- [batch, height, width, channels]
input_list = []
input_list.append(y)
input_list = np.asarray(input_list).astype(np.float32)
# ======================
# MODIFY DROPOUT HERE FROM JSON FILE
# CHANGED VALUES FROM 0.0, 0.5, 1.0 -- same prediction score
# ======================
# run and get output
if self.dropout_training_Flag == 1:
output = sess.run(y_op, feed_dict={x_inp: input_list, tflag_op: True, dropout_prob_op: self.dropout_prob_testing})
else:
output = sess.run(y_op, feed_dict={x_inp: input_list, tflag_op: False})
tmpOut = cv2.resize(output[0,:,:,0], (self.original_input_image_width, self.original_input_image_height), interpolation = cv2.INTER_LANCZOS4)
prediction = np.asarray((tmpOut > 0.5))
labels = np.asarray((lab > 0))
EPS = 0.0000001
#output_original = cv2.resize(output[0,:,:,0], (original_input_image_width, original_input_image_height), interpolation = cv2.INTER_LANCZOS4)
loss = 2.0 * np.sum(labels * prediction, axis=(0, 1)) / (np.sum(labels ** 2 + prediction ** 2, axis=(0, 1)) + EPS)
loss_for_each_image.append(loss)
#place slice in final_vol
#print(curr_dataset_label_numbers[elem_idx][1])
#print(type(curr_dataset_label_numbers[elem_idx][1]))
final_vol[:,:,curr_dataset_label_numbers[elem_idx][1] - 1] = np.asarray(prediction*255.0).astype(np.uint8)
# dice mean over dataset
dice_mean_for_dataset = np.mean(loss_for_each_image)
print(dataset_name, dice_mean_for_dataset)
self.diceScore_for_each_dataset.append(dice_mean_for_dataset)
self.list_dataset_name.append(dataset_name)
This is the code for the inputs/outputs:
#===============
def create_placeholders(self):
self.placeholders['images'] = tf.placeholder(
shape=[None] + self.network_input_size + [self.network_input_channels],
name='images',
dtype=tf.float32
)
self.placeholders['labels'] = tf.placeholder(
shape=[None] + self.network_input_size + [self.network_output_channels],
name='labels',
dtype=tf.float32
)
self.placeholders['is_training'] = tf.placeholder(
shape=[],
name='is_training',
dtype=tf.bool
)
# dropout is included
if self.dropout_training_Flag == 1:
self.placeholders['dropout_prob'] = tf.placeholder(
shape=[],
name='dropout_prob',
dtype=tf.float32
)
#===============
def create_outputs(self):
if self.network_name == 'UNet':
print('\n')
print('Training UNet')
# dropout is included
if self.dropout_training_Flag == 1:
# train with dropout
unet_output = unet_dropout(
self.placeholders['images'],
self.placeholders['is_training'],
self.placeholders['dropout_prob'],
self.network_output_channels
)
if self.network_output_channels == 1:
self.outputs['sigmoid'] = unet_output
else:
self.outputs['sigmoid'] = unet_output
This is the code for my model:
#===============
def batch_norm_relu(inputs, is_training):
net = slim.batch_norm(inputs, is_training=is_training)
net = tf.nn.relu(net)
return net
#===============
def dropout (input, keep_prob, is_training):
if is_training == True:
dropout = tf.nn.dropout(input, keep_prob)
else:
dropout = input
return dropout
#===============
def model(inputs, is_training, keep_prob, num_classes):
with tf.variable_scope("model", reuse=tf.AUTO_REUSE):
base_num_kernels = 64
# =================================
# encoder
# 256
x = conv2d_fixed_padding(inputs=inputs, filters=base_num_kernels, kernel_size=3, stride=1)
x = batch_norm_relu(x, is_training)
x = conv2d_fixed_padding(inputs=x, filters=base_num_kernels, kernel_size=3, stride=1)
x = batch_norm_relu(x, is_training)
output_b1 = x
output_list_b1 = [x]
output_b1 = dropout(output_b1, keep_prob, is_training)
output_b1 = tf.layers.max_pooling2d(inputs=output_b1, pool_size=2, strides=2, padding='SAME')
# =================================
# 128
x = conv2d_fixed_padding(inputs=output_b1, filters=2*base_num_kernels, kernel_size=3, stride=1)
x = batch_norm_relu(x, is_training)
x = conv2d_fixed_padding(inputs=x, filters=2*base_num_kernels, kernel_size=3, stride=1)
x = batch_norm_relu(x, is_training)
output_b2 = x
output_list_b2 = [x]
output_b2 = dropout(output_b2, keep_prob, is_training)
# =================================
# decoder
# 128 -> 256
output_b3 = conv2d_transpose(output_b2, kernel_size=2, output_channels=base_num_kernels)
output_b4 = tf.concat([output_b3, x], axis=3)
# =================================
# 256
conv_final = conv2d_fixed_padding(inputs=output_b4, filters=base_num_kernels, kernel_size=3, stride=1)
conv_final = batch_norm_relu(conv_final, is_training)
conv_final = conv2d_fixed_padding(inputs=conv_final, filters=base_num_kernels, kernel_size=3, stride=1)
conv_final = batch_norm_relu(conv_final, is_training)
# =================================
# output
outputs = conv2d_fixed_padding(inputs=conv_final, filters=num_classes, kernel_size=3, stride=1)
if num_classes == 1:
outputs = tf.nn.sigmoid(outputs)
else:
h = outputs.get_shape().as_list()[1]
w = outputs.get_shape().as_list()[2]
outputs_reshaped = tf.reshape(outputs, np.asarray([-1, num_classes]))
outputs_final = tf.nn.softmax(outputs_reshaped)
outputs = tf.reshape(outputs_final, np.asarray([-1, h, w, num_classes]))
return outputs
This is the way that I save the network weights:
#===============
def __create_summary_manager(self):
self.saver = Saver(
self.sess,
self.placeholders,
self.outputs,
self.savepath
)
#===============
import tensorflow as tf
class Saver(object):
def __init__(self, sess, input_dict, output_dict, path):
self.sess = sess
self.input_dict = input_dict
self.output_dict = output_dict
self.path = path
self.iteration = 0
self.input_dict_info = {}
self.output_dict_info = {}
for key in input_dict.keys():
self.input_dict_info[key] = \
tf.saved_model.utils.build_tensor_info(
self.input_dict[key]
)
for key in output_dict.keys():
self.output_dict_info[key] = \
tf.saved_model.utils.build_tensor_info(
self.output_dict[key]
)
self.prediction_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs=self.input_dict_info,
outputs=self.output_dict_info)
)
def save(self, iteration_val):
self.iteration += 1
export_path = os.path.join(
tf.compat.as_bytes(self.path),
tf.compat.as_bytes(str(iteration_val))
)
self.builder = tf.saved_model.builder.SavedModelBuilder(export_path)
self.builder.add_meta_graph_and_variables(
self.sess, [tf.saved_model.tag_constants.SERVING],
signature_def_map={
'prediction': self.prediction_signature,
}
)
self.builder.save()
I'm trying to implement model for triplet loss using FacenetModel. I used the Facenet implementation provided in coursera's assignments.
Whenever I compile the model I am getting this error:
ValueError: No data provided for "FaceRecoModel". Need data for each key in: ['FaceRecoModel', 'FaceRecoModel', 'FaceRecoModel']
my code:
def batch_generator(batch_size = 64):
while True:
pos = positiveImg[np.random.choice(len(positiveImg), batch_size)]
neg = negativeImg[np.random.choice(len(negativeImg), batch_size)]
anc = anchorsImg[np.random.choice(len(anchorsImg), batch_size)]
x_data = {'inp1': anc,
'inp2': pos,
'inp3': neg
}
y_data = {'y1': np.zeros((64,0)),
'y2': np.zeros((64,0)),
'y3': np.zeros((64,0))}
yield (x_data, y_data)
def triplet_loss(y_true, y_pred):
anchor, positive, negative = y_pred[0], y_pred[1], y_pred[2]
pos_dist = tf.reduce_sum(tf.square(tf.subtract(anchor, positive)), axis=-1)
neg_dist = tf.reduce_sum(tf.square(tf.subtract(anchor, negative)), axis=-1)
basic_loss = tf.add(tf.subtract(pos_dist, neg_dist), 0.2)
loss = tf.reduce_sum(tf.maximum(basic_loss, 0.0))
return loss
def getModels():
FRmodel = keras.models.load_model('FR.h5', custom_objects={'triplet_loss': triplet_loss})
inp1 = Input((3, 96, 96), name= 'inp1')
inp2 = Input((3, 96, 96), name= 'inp2')
inp3 = Input((3, 96, 96), name= 'inp3')
pred1 = FRmodel(inp1)
pred2 = FRmodel(inp2)
pred3 = FRmodel(inp3)
inputs = [inp1, inp2, inp3]
outputs = [pred1, pred2, pred3]
model = keras.models.Model(inputs=[inp1, inp2, inp3], outputs= [pred1, pred2, pred3])
return FRmodel, model
generator = batch_generator(64)
FRmodel, my_model = getModels()
my_model.compile(optimizer = 'adam', loss = triplet_loss, metrics = ['accuracy'])
my_model.fit_generator(generator, epochs=5,steps_per_epoch=30)
Summary of pretrained Facenet model:
FRmodel.summary() : https://codeshare.io/arxmev
my_model.summary() : https://codeshare.io/arx3N6
Found the solution on coursera's discussion forum. It is little tricky.I have to add euclidean distance of triplet loss in keras layer wrapper using Lambda. According to documentation:
Wraps arbitrary expression as a Layer object.
new implementation:
`
def triplet_loss_v2(y_true, y_pred):
positive, negative = y_pred[:,0,0], y_pred[:,1,0]
margin = K.constant(0.2)
loss = K.mean(K.maximum(K.constant(0), positive - negative + margin))
return loss # shape = [1]
def euclidean_distance(vects):
x, y = vects # shape = [batch_size, 2, 1]
dist = K.sqrt(K.maximum(K.sum(K.square(x - y), axis=1, keepdims=True), K.epsilon()))
return dist # shape = [batch_size, 1]
FRmodel = faceRecoModel(input_shape=(3, 96, 96))
load_weights_from_FaceNet(FRmodel)
for layer in FRmodel.layers[0: 80]:
layer.trainable = False
input_shape=(3, 96, 96)
anchor = Input(shape=input_shape, name = 'anchor')
anchorPositive = Input(shape=input_shape, name = 'anchorPositive')
anchorNegative = Input(shape=input_shape, name = 'anchorNegative')
anchorCode = FRmodel(anchor)
anchorPosCode = FRmodel(anchorPositive)
anchorNegCode = FRmodel(anchorNegative)
positive_dist = Lambda(euclidean_distance, name='pos_dist')([anchorCode, anchorPosCode])
negative_dist = Lambda(euclidean_distance, name='neg_dist')([anchorCode, anchorNegCode])
stacked_dists = Lambda(lambda vects: K.stack(vects, axis=1), name='stacked_dists')([positive_dist, negative_dist]) # shape = [batch_size, 2, 1]
tripletModel = Model([anchor, anchorPositive, anchorNegative], stacked_dists, name='triple_siamese')
tripletModel.compile(optimizer = 'adadelta', loss = triplet_loss_v2, metrics = None)
gen = batch_generator(64)
tripletModel.fit_generator(gen, epochs=1,steps_per_epoch=5)`
I want to use part of a the tensor in the output of lstm layer, but don't know how to do it correctly.
My purpose is tell LSTM layer the "real" length of its input sequence.
Here is my attempt, but it fails.
Isthere anyone who can help solve this problem and explain the details, thanks a lot~
input_spectrogram = Input(shape=(64,500,1))
input_length = Input(shape=(1,))
cnn1 = Conv2D(filters = 64, kernel_size = (1,4),input_shape=(64,500, 1),padding = 'same',strides = 1,activation = 'relu',name='conv1')(input_spectrogram)
maxpooling1 = MaxPooling2D(pool_size = (1,4),name='maxpooling1')(cnn1)
bn1 = BatchNormalization(name='BN1')(maxpooling1)
cnn2 = Conv2D(filters = 128, kernel_size = (64,1),strides = 1,activation ='relu',name='conv2')(bn1)
maxpooling2 = MaxPooling2D(pool_size = (1,2),name='maxpooling2')(cnn2)
reshape = Reshape((62,128))(maxpooling2)
lstm1 = LSTM(128,return_sequences = True,recurrent_dropout=0.3,name='lstm1')(reshape) #output:(None,62,128)
softmax_in = Lambda(lambda x:x[0][x[1],:])([lstm1,input_length])
softmax_ = Dense(10,activation='softmax',name='softmax_')(softmax_in)
seq = Model(inputs=input_spectrogram, outputs=[softmax_])
seq.compile(loss='categorical_crossentropy', optimizer='adadelta',metrics=['accuracy'])
Seems to be indexing with tensor is not fully supported (see discussion here: https://github.com/tensorflow/tensorflow/issues/206#issuecomment-158435464).
Does it work for you to perform indexing with constant instead?
input_spectrogram = Input(shape=(64,500,1))
input_length = Input(shape=(1,))
cnn1 = Conv2D(filters = 64, kernel_size = (1,4),input_shape=(64,500, 1),padding = 'same',strides = 1,activation = 'relu',name='conv1')(input_spectrogram)
maxpooling1 = MaxPooling2D(pool_size = (1,4),name='maxpooling1')(cnn1)
bn1 = BatchNormalization(name='BN1')(maxpooling1)
cnn2 = Conv2D(filters = 128, kernel_size = (64,1),strides = 1,activation ='relu',name='conv2')(bn1)
maxpooling2 = MaxPooling2D(pool_size = (1,2),name='maxpooling2')(cnn2)
reshape = Reshape((62,128))(maxpooling2)
lstm1 = LSTM(128,return_sequences = True,recurrent_dropout=0.3,name='lstm1')(reshape) #output:(None,62,128)
softmax_in = Lambda(lambda x:x[:,5])(lstm1)
softmax_ = Dense(10,activation='softmax',name='softmax_')(softmax_in)
seq = Model(inputs=input_spectrogram, outputs=[softmax_])
seq.compile(loss='categorical_crossentropy', optimizer='adadelta',metrics=['accuracy'])
now it is feasible, so how to use the "real_length" from an input layer?