I am new to tensorflow, and I think I got the right answer, but I am missing something minimal, that I cant find online. I hope someone send me a reference or leads me to what I am missing.
import tensorflow as tf
import numpy as np
from tensorflow.examples.tutorials.mnist import input_data
batch_size = 128
test_size = 256
def init_weights(shape):
return tf.Variable(tf.random_normal(shape, stddev=0.01))
def model(X, w, w2, w3, w4, w_o, p_keep_conv, p_keep_hidden):
l1a = tf.nn.relu(tf.nn.conv2d(X, w, # l1a shape=(?, 28, 28, 32)
strides=[1, 1, 1, 1], padding='SAME'))
l1 = tf.nn.max_pool(l1a, ksize=[1, 2, 2, 1], # l1 shape=(?, 14, 14, 32)
strides=[1, 2, 2, 1], padding='SAME')
l1 = tf.nn.dropout(l1, p_keep_conv)
l2a = tf.nn.relu(tf.nn.conv2d(l1, w2, # l2a shape=(?, 14, 14, 64)
strides=[1, 1, 1, 1], padding='SAME'))
l2 = tf.nn.max_pool(l2a, ksize=[1, 2, 2, 1], # l2 shape=(?, 7, 7, 64)
strides=[1, 2, 2, 1], padding='SAME')
l2 = tf.nn.dropout(l2, p_keep_conv)
l3a = tf.nn.relu(tf.nn.conv2d(l2, w3, # l3a shape=(?, 7, 7, 128)
strides=[1, 1, 1, 1], padding='SAME'))
l3 = tf.nn.max_pool(l3a, ksize=[1, 2, 2, 1], # l3 shape=(?, 4, 4, 128)
strides=[1, 2, 2, 1], padding='SAME')
l3 = tf.reshape(l3, [-1, w4.get_shape().as_list()[0]]) # reshape to (?, 2048)
l3 = tf.nn.dropout(l3, p_keep_conv)
l4 = tf.nn.relu(tf.matmul(l3, w4))
l4 = tf.nn.dropout(l4, p_keep_hidden)
pyx = tf.matmul(l4, w_o)
return pyx
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
trX, trY, teX, teY = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels
trX = trX.reshape(-1, 28, 28, 1) # 28x28x1 input img
teX = teX.reshape(-1, 28, 28, 1) # 28x28x1 input img
X = tf.placeholder("float", [None, 28, 28, 1])
Y = tf.placeholder("float", [None, 10])
w = init_weights([3, 3, 1, 32]) # 3x3x1 conv, 32 outputs
w2 = init_weights([3, 3, 32, 64]) # 3x3x32 conv, 64 outputs
w3 = init_weights([3, 3, 64, 128]) # 3x3x32 conv, 128 outputs
w4 = init_weights([128 * 4 * 4, 625]) # FC 128 * 4 * 4 inputs, 625 outputs
w_o = init_weights([625, 10]) # FC 625 inputs, 10 outputs (labels)
p_keep_conv = tf.placeholder("float")
p_keep_hidden = tf.placeholder("float")
py_x = model(X, w, w2, w3, w4, w_o, p_keep_conv, p_keep_hidden)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=py_x, labels=Y))
train_op = tf.train.RMSPropOptimizer(0.001, 0.9).minimize(cost)
predict_op = tf.argmax(py_x, 1)
# Launch the graph in a session
saver = tf.train.Saver()
with tf.Session() as sess:
# you need to initialize all variables
tf.global_variables_initializer().run()
for i in range(100):
training_batch = zip(range(0, len(trX), batch_size),
range(batch_size, len(trX)+1, batch_size))
for start, end in training_batch:
sess.run(train_op, feed_dict={X: trX[start:end], Y: trY[start:end],
p_keep_conv: 0.8, p_keep_hidden: 0.5})
test_indices = np.arange(len(teX)) # Get A Test Batch
np.random.shuffle(test_indices)
test_indices = test_indices[0:test_size]
print(i, np.mean(np.argmax(teY[test_indices], axis=1) ==
sess.run(predict_op, feed_dict={X: teX[test_indices],
Y: teY[test_indices],
p_keep_conv: 1.0,
p_keep_hidden: 1.0})))
save_path = saver.save(sess, "tmp/model.ckpt")
print("Model saved in file: %s" % save_path)
After all of this, now I am trying to predict a single image from this array just as an example (I know its not a proper test), to give me the class using:
with tf.Session() as sess:
# Restore variables from disk.
saver.restore(sess, "tmp/model.ckpt")
print "...Model Loaded..."
prediction=tf.argmax(predict_op,1)
print prediction.eval(feed_dict={X: teX[2].reshape(1,28,28,1)}, session=sess)
But im getting this error:
InvalidArgumentError: You must feed a value for placeholder tensor
'Placeholder_3' with dtype
This previous problem has been solved by adding p_keep_conv: 1.0, p_keep_hidden: 1.0 to the dict.
After this another issue appeared:
InvalidArgumentError Traceback (most recent call last)
<ipython-input-91-3e9ead14a8b3> in <module>()
4 print "...Model Loaded..."
5 prediction=tf.argmax(predict_op,1)
----> 6 classification = sess.run(tf.argmax(predict_op, 1), feed_dict={X: teX[3].reshape(1,28,28,1),p_keep_conv: 1.0,p_keep_hidden: 1.0})
7
....
InvalidArgumentError: Expected dimension in the range [-1, 1), but got 1
[[Node: ArgMax_21 = ArgMax[T=DT_INT64, Tidx=DT_INT32, output_type=DT_INT64, _device="/job:localhost/replica:0/task:0/device:CPU:0"](ArgMax/_37, ArgMax_21/dimension)]]
I'm summing up what we said in the comments in this answer.
Placeholder error:
Your prediction.eval() call has a feed_dict that doesn't contain a value for p_keep_conv and p_keep_hidden. Note that, since you don't provide a name=... argument whe defining your placholders, they get the default name Placeholder_N which is what the error message shows. It's a good practice to always give a meaningful name to variables, constants and placeholders in order to make debugging easier.
Argmax expected dimension:
tf.argmax's definition states:
axis: A Tensor. Must be one of the following types: int32, int64.
int32, 0 <= axis < rank(input). Describes which axis of the input
Tensor to reduce across.
It seems, then, that the only way to run argmax on the last axis of the tensor is by giving it axis=-1, because of the "strictly less than" sign in the definition of the function (I don't understand why they made this design choice).
Related
So I want to classify some (3, 50, 50) pictures. First I loaded the dataset from the file without a dataloader or batches, it worked. Now, after adding both things I get that error:
RuntimeError: multi-target not supported at /pytorch/aten/src/THCUNN/generic/ClassNLLCriterion.cu:15
I found a lot of answers in the internet, mostly to use target.squeeze(1) but it doesn´t work for me.
My target-batch looks like following:
tensor([[1, 0],
[1, 0],
[1, 0],
[1, 0],
[1, 0],
[1, 0],
[1, 0],
[1, 0]], device='cuda:0')
Shouldn't that be okay?
Here the full code (notice that Im only creating the structure of the model on which Im going to apply the full and correct dataset afterwards, because I dont have the full data yet, only 32 pictures and no labels, thats why I added torch.tensor([1, 0]) as a placeholder for all labels):
import torch
import torch.utils.data
import torch.nn as nn
import torch.nn.functional as F
import torch.optim
from torch.autograd import Variable
import numpy as np
from PIL import Image
class Model(nn.Module):
def __init__(self):
super(Model, self).__init__()
# model structur:
self.conv1 = nn.Conv2d(3, 10, kernel_size=(5,5), stride=(1,1))
self.conv2 = nn.Conv2d(10, 20, kernel_size=(5,5), stride=(1,1)) # with mapool: output = 20 * (9,9) feature-maps -> flatten
self.fc1 = nn.Linear(20*9*9, 250)
self.fc2 = nn.Linear(250, 100)
self.fc3 = nn.Linear(100, 2)
def forward(self, x):
# conv layers
x = F.relu(self.conv1(x)) # shape: 1, 10, 46, 46
x = F.max_pool2d(x, 2, 2) # shape: 1, 10, 23, 23
x = F.relu(self.conv2(x)) # shape: 1, 20, 19, 19
x = F.max_pool2d(x, 2, 2) # shape: 1, 20, 9, 9
# flatten to dense layer:
x = x.view(-1, 20*9*9)
# dense layers
x = F.relu(self.fc1(x))
x = F.relu(self.fc2(x))
output = F.log_softmax(self.fc3(x), dim=1)
return output
class Run:
def __init__(self, epochs, learning_rate, dropout, momentum):
# load model
self.model = Model().cuda()
# hyperparameters:
self.epochs = epochs
self.learning_rate = learning_rate
self.dropout = dropout
def preporcessing(self):
dataset_folder = "/media/theodor/hdd/Programming/BWKI/dataset/bilder/"
dataset = []
for i in range(0, 35):
sample_image = Image.open(dataset_folder + str(i) + ".png")
data = torch.from_numpy(np.array(sample_image)).type("torch.Tensor").reshape(3, 50, 50)
target = torch.tensor([[1, 0]])
sample = (data, target)
dataset.append(sample)
train_loader = torch.utils.data.DataLoader(dataset, batch_size=8)
return train_loader
def train(self):
train_set = self.preporcessing()
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(self.model.parameters(), lr=self.learning_rate)
for epoch in range(self.epochs):
epoch_loss = 0
for i, data in enumerate(train_set, 0):
sample, target = data
# set data as cuda varibale
sample = Variable(sample.float().cuda())
target = Variable(target.cuda())
# initialize optimizer
optimizer.zero_grad()
# predict
output = self.model(sample)
# backpropagation
print(output, target.squeeze(1))
loss = criterion(output, target.squeeze(1)) # ERROR MESSAGE: RuntimeError: multi-target not supported at /pytorch/aten/src/THCUNN/generic/ClassNLLCriterion.cu:15
loss.backward()
optimizer.step()
epoch_loss += loss.item()
print("loss after epoch [", epoch, "|", self.epochs, "] :", epoch_loss)
run = Run(10, 0.001, 0.5, 0.9)
run.train()
So I expected it to start training (of course not learning anything because the labels are wrong).
For nn.CrossEntropyLoss the target has to be a single number from the interval [0, #classes] instead of a one-hot encoded target vector. Your target is [1, 0], thus PyTorch thinks you want to have multiple labels per input which is not supported.
Replace your one-hot-encoded targets:
[1, 0] --> 0
[0, 1] --> 1
When I build the FCN for segmentation, I want the images to keep the original size of input data, so I use the fully convolution layers. When I choose the fixed input size, such as (224, 224), the transpose conv works fine. However, when I changed the code of using (224, 224) to (h, w), I meet the following error. I googled before, but I didn't figure it out. Can anyone help me? Thanks.
Error information:
InvalidArgumentError (see above for traceback): Conv2DSlowBackpropInput: Size
of out_backprop doesn't match computed: actual = 62, computed =
63spatial_dim: 2 input: 500 filter: 16 output: 62 stride: 8 dilation: 1
[[Node: deconv_layer/conv2d_transpose_2 =
Conv2DBackpropInput[T=DT_FLOAT, data_format="NCHW", dilations=[1, 1, 1, 1],
padding="SAME", strides=[1, 1, 8, 8], use_cudnn_on_gpu=true,
_device="/job:localhost/replica:0/task:0/device:GPU:0"]
(deconv_layer/conv2d_transpose_2-0-VecPermuteNHWCToNCHW-
LayoutOptimizer/_1961, deconv_layer/deconv3/kernel/read,
deconv_layer/Add_1)]]
[[Node: losses/_2091 = _Recv[client_terminated=false,
recv_device="/job:localhost/replica:0/task:0/device:CPU:0",
send_device="/job:localhost/replica:0/task:0/device:GPU:0",
send_device_incarnation=1, tensor_name="edge_4480_losses",
tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]
()]]
Code:
with tf.variable_scope("deconv_layer"):
deconv_shape1 = block2.get_shape()
W_t1 = deconv_utils.weight_variable([4, 4, deconv_shape1[3].value, 2048],
name="deconv1/kernel")
b_t1 = deconv_utils.bias_variable([deconv_shape1[3].value],
name="deconv1/biases")
deconv_t1 = deconv_utils.conv2d_transpose_strided(block4, W_t1, b_t1,
output_shape=tf.shape(block2))
fuse1 = tf.add(deconv_t1, block2)
print("deconv_t1: ", deconv_t1.shape)
print("fuse_1: ", fuse1.shape)
tf.identity(fuse1, name="fuse1")
deconv_shape2 = block1.get_shape()
W_t2 = deconv_utils.weight_variable([4, 4, deconv_shape2[3].value,
deconv_shape1[3].value], name="deconv2/kernel")
b_t2 = deconv_utils.bias_variable([deconv_shape2[3].value],
name="deconv2/biases")
deconv_t2 = deconv_utils.conv2d_transpose_strided(fuse1, W_t2, b_t2,
output_shape=tf.shape(block1))
fuse2 = tf.add(deconv_t2, block1)
print("deconv_t2: ", deconv_t2.shape)
print("fuse2: ", fuse2.shape)
tf.identity(fuse2, name="fuse2")
shape = tf.shape(features)
deconv_shape3 = tf.stack([shape[0], shape[1], shape[2], num_classes])
W_t3 = deconv_utils.weight_variable([16, 16, num_classes,
deconv_shape2[3].value], name="deconv3/kernel")
b_t3 = deconv_utils.bias_variable([num_classes], name="deconv3/biases")
deconv_t3 = deconv_utils.conv2d_transpose_strided(fuse2, W_t3, b_t3,
output_shape=deconv_shape3, stride=8)
print("deconv_t3: ", deconv_t3.shape)
The version with out custom functions is here:
with tf.variable_scope("deconv_layer"):
deconv1_shape = block2.get_shape()
shape1 = [4, 4, deconv1_shape[3].value, 2048]
deconv1_kernel = tf.Variable(initial_value=tf.truncated_normal(shape1,
stddev=0.02),
trainable=True,
name="deconv1/kernel")
deconv1 = tf.nn.conv2d_transpose(value=block4,
filter=deconv1_kernel,
# output_shape=[BATCH_SIZE,
tf.shape(block2)[1], tf.shape(block2)[2], 512],
output_shape=tf.shape(block2),
strides=[1, 2, 2, 1],
padding='SAME',
data_format='NHWC'
)
print('deconv1', deconv1.shape)
fuse1 = tf.add(deconv1, block2) # fuse1 = pool4 + deconv2(pool5)
tf.identity(fuse1, name="fuse1")
deconv2_shape = block1.get_shape()
shape2 = [4, 4, deconv2_shape[3].value, deconv1_shape[3].value]
deconv2_kernel = tf.Variable(initial_value=tf.truncated_normal(shape2,
stddev=0.02),
trainable=True,
name="deconv2/kernel")
deconv2 = tf.nn.conv2d_transpose(value=fuse1,
filter=deconv2_kernel,
output_shape=tf.shape(block1),
strides=[1, 2, 2, 1],
padding='SAME',
data_format='NHWC'
)
print('deconv2', deconv2.shape)
fuse2 = tf.add(deconv2, block1)
tf.identity(fuse2, name="fuse2")
deconv3_shape = tf.stack([tf.shape(features)[0], tf.shape(features)[1],
tf.shape(features)[2], num_classes])
shape3 = [16, 16, num_classes, deconv2_shape[3].value]
deconv_final_kernel = tf.Variable(initial_value=tf.truncated_normal(shape3, stddev=0.02),
trainable=True,
name="deconv3/kernel")
seg_logits = tf.nn.conv2d_transpose(value=fuse2,
filter=deconv_final_kernel,
output_shape=deconv3_shape,
strides=[1, 8, 8, 1],
padding='SAME',
data_format='NHWC')
The conv Net and Deconv Net in FCN, which are built by different structures, are maybe not consistent with each other. In this case, the conv net use conv with padding='VALID', while the deconv net uses all conv_transpose with padding='SAME. Thus the shapes are not the same, which causes the problem above.
This is because of your stride > 1. The calculation can not be correct at all time. This GitHub post explains it.
I had similar issue while trying to replicate pytorch's transposeconv2d function in tensorflow. I was trying to do padding on input before passing to the conv2d_transpose() function and doing padding again on the deconvolved output. This was the reason why graph was initialized properly but there was error in calculating the gradients. I solved the error by removing all manual paddings and changing padding="SAME" inside the function. I guess this is handeled internally in the function. Correct me if I am wrong. I don't know how much this affects the actual output.
I'm new to Tensorflow. I worked in Caffe previously. I'm trying to implement http://cvlab.cse.msu.edu/pdfs/Tai_Yang_Liu_CVPR2017.pdf in Tensorflow.
I'm having trouble with variables in Tensorflow, despite having them initialized. I tried using tf.get_variable instead of tf.Variable, but this didn't work. And setting initializer=tf.contrib.layers.xavier_initializer() did nothing.
My code:
import tensorflow as tf
import sys, os
import numpy as np
global xseed
def get_model(inp, train):
#create architecture
#db input, 128 batch size
with tf.name_scope('input'):
inl = tf.reshape(inp, [-1, 31, 31, 1])
with tf.name_scope('batchnorm_scale_relu'):
#batchnorm,scale,relu
normed = tf.contrib.layers.batch_norm(inl, is_training=train)
alpha = tf.Variable(tf.truncated_normal((1,), stddev=0.1), trainable=True, name="alpha")
beta = tf.Variable(tf.truncated_normal((1,), stddev=0.1), trainable=True, name="beta")
scaled = alpha * normed + beta
relud = tf.nn.relu(scaled, name="relu1")
with tf.name_scope('conv1'):
#??? padding ??? is correct
padded_input = tf.pad(relud, [[0, 0], [1, 1], [1, 1], [0, 0]], "CONSTANT")
#weight initializer msra
convolved = tf.layers.conv2d(inputs=padded_input, filters=128, kernel_size=(3, 3), strides=(1, 1), padding="VALID",
kernel_initializer=tf.contrib.layers.variance_scaling_initializer(seed=xseed))
with tf.name_scope('batchnorm_scale_relu_1a'):
#batchnorm,scale,relu
normed_1a = tf.contrib.layers.batch_norm(convolved, is_training=train)
alpha_1a = tf.Variable(tf.truncated_normal((1,), stddev=0.1), trainable=True, name="alpha_1a")
beta_1a = tf.Variable(tf.truncated_normal((1,), stddev=0.1), trainable=True, name="beta_1a")
scaled_1a = alpha_1a * normed_1a + beta_1a
relud_1a = tf.nn.relu(scaled_1a, name="relu1_1a")
with tf.name_scope('conv1_1a'):
padded_input_1a = tf.pad(relud_1a, [[0, 0], [1, 1], [1, 1], [0, 0]], "CONSTANT")
#weight initializer msra
convolved_1a = tf.layers.conv2d(inputs=padded_input_1a, filters=128, kernel_size=(3, 3), strides=(1, 1), padding="VALID",
kernel_initializer=tf.contrib.layers.variance_scaling_initializer(seed=xseed))
return convolved_1a
xseed = None
with tf.device("cpu"):
inp = tf.placeholder(tf.float32, [None, 31, 31, 1])
init_op = tf.global_variables_initializer()
init_op2 = tf.local_variables_initializer()
with tf.Session(config=tf.ConfigProto()) as sess:
m = get_model(inp, True)
sess.run(init_op)
sess.run(init_op2)
print(sess.run(tf.report_uninitialized_variables()))
res = sess.run(m, feed_dict={ inp: np.zeros((1, 31, 31, 1))})
And it reports uninitialized variables:
['BatchNorm/beta' 'BatchNorm/moving_mean' 'BatchNorm/moving_variance' 'batchnorm_scale_relu/alpha' 'batchnorm_scale_relu/beta' 'conv2d/kernel' 'conv2d/bias' 'BatchNorm_1/beta' 'BatchNorm_1/moving_mean' 'BatchNorm_1/moving_variance' 'batchnorm_scale_relu_1a/alpha_1a' 'batchnorm_scale_relu_1a/beta_1a' 'conv2d_1/kernel' 'conv2d_1/bias']
And gives me an exception when evaluating convolved tensor:
FailedPreconditionError (see above for traceback): Attempting to use uninitialized value batchnorm_scale_relu_1a/alpha_1a
[[Node: batchnorm_scale_relu_1a/alpha_1a/read = Identity[T=DT_FLOAT, _class=["loc:#batchnorm_scale_relu_1a/alpha_1a"], _device="/job:localhost/replica:0/task:0/device:CPU:0"](batchnorm_scale_relu_1a/alpha_1a)]]
Please help me.
Solved it myself. Replaced this:
with tf.device("cpu"):
inp = tf.placeholder(tf.float32, [None, 31, 31, 1])
init_op = tf.global_variables_initializer()
init_op2 = tf.local_variables_initializer()
with tf.Session(config=tf.ConfigProto()) as sess:
m = get_model(inp, True)
sess.run(init_op)
sess.run(init_op2)
print(sess.run(tf.report_uninitialized_variables()))
res = sess.run(m, feed_dict={ inp: np.zeros((1, 31, 31, 1))})
With this:
with tf.device("cpu"):
inp = tf.placeholder(tf.float32, [None, 31, 31, 1])
with tf.Session(config=tf.ConfigProto()) as sess:
m = get_model(inp, True)
sess.run(tf.initialize_all_variables())
res = sess.run(tf.report_uninitialized_variables())
#print(res) -- outputs [] (none)
res = sess.run(m, feed_dict={ inp: np.zeros((1, 31, 31, 1))})
print(res)
The whole point is tf.initialize_all_variables() instead of tf.global_variables_initializer()
I am attempting to create a deep CNN that can classify each individual pixel in an image. I am replicating architecture from the image below taken from this paper. In the paper it is mentioned that deconvolutions are used so that any size of input is possible. This can be seen in the image below.
Github Repository
Currently, I have hard coded my model to accept images of size 32x32x7, but I would like to accept any size of input. What changes would I need to make to my code to accept variable sized input?
x = tf.placeholder(tf.float32, shape=[None, 32*32*7])
y_ = tf.placeholder(tf.float32, shape=[None, 32*32*7, 3])
...
DeConnv1 = tf.nn.conv3d_transpose(layer1, filter = w, output_shape = [1,32,32,7,1], strides = [1,2,2,2,1], padding = 'SAME')
...
final = tf.reshape(final, [1, 32*32*7])
W_final = weight_variable([32*32*7,32*32*7,3])
b_final = bias_variable([32*32*7,3])
final_conv = tf.tensordot(final, W_final, axes=[[1], [1]]) + b_final
Dynamic placeholders
Tensorflow allows to have multiple dynamic (a.k.a. None) dimensions in placeholders. The engine won't be able to ensure correctness while the graph is built, hence the client is responsible for feeding the correct input, but it provides a lot of flexibility.
So I'm going from...
x = tf.placeholder(tf.float32, shape=[None, N*M*P])
y_ = tf.placeholder(tf.float32, shape=[None, N*M*P, 3])
...
x_image = tf.reshape(x, [-1, N, M, P, 1])
to...
# Nearly all dimensions are dynamic
x_image = tf.placeholder(tf.float32, shape=[None, None, None, None, 1])
label = tf.placeholder(tf.float32, shape=[None, None, 3])
Since you intend to reshape the input to 5D anyway, so why don't use 5D in x_image right from the start. At this point, the second dimension of label is arbitrary, but we promise tensorflow that it will match with x_image.
Dynamic shapes in deconvolution
Next, the nice thing about tf.nn.conv3d_transpose is that its output shape can be dynamic. So instead of this:
# Hard-coded output shape
DeConnv1 = tf.nn.conv3d_transpose(layer1, w, output_shape=[1,32,32,7,1], ...)
... you can do this:
# Dynamic output shape
DeConnv1 = tf.nn.conv3d_transpose(layer1, w, output_shape=tf.shape(x_image), ...)
This way the transpose convolution can be applied to any image and the result will take the shape of x_image that was actually passed in at runtime.
Note that static shape of x_image is (?, ?, ?, ?, 1).
All-Convolutional network
Final and most important piece of the puzzle is to make the whole network convolutional, and that includes your final dense layer too. Dense layer must define its dimensions statically, which forces the whole neural network fix input image dimensions.
Luckily for us, Springenberg at al describe a way to replace an FC layer with a CONV layer in "Striving for Simplicity: The All Convolutional Net" paper. I'm going to use a convolution with 3 1x1x1 filters (see also this question):
final_conv = conv3d_s1(final, weight_variable([1, 1, 1, 1, 3]))
y = tf.reshape(final_conv, [-1, 3])
If we ensure that final has the same dimensions as DeConnv1 (and others), it'll make y right the shape we want: [-1, N * M * P, 3].
Combining it all together
Your network is pretty large, but all deconvolutions basically follow the same pattern, so I've simplified my proof-of-concept code to just one deconvolution. The goal is just to show what kind of network is able to handle images of arbitrary size. Final remark: image dimensions can vary between batches, but within one batch they have to be the same.
The full code:
sess = tf.InteractiveSession()
def conv3d_dilation(tempX, tempFilter):
return tf.layers.conv3d(tempX, filters=tempFilter, kernel_size=[3, 3, 1], strides=1, padding='SAME', dilation_rate=2)
def conv3d(tempX, tempW):
return tf.nn.conv3d(tempX, tempW, strides=[1, 2, 2, 2, 1], padding='SAME')
def conv3d_s1(tempX, tempW):
return tf.nn.conv3d(tempX, tempW, strides=[1, 1, 1, 1, 1], padding='SAME')
def weight_variable(shape):
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial)
def bias_variable(shape):
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial)
def max_pool_3x3(x):
return tf.nn.max_pool3d(x, ksize=[1, 3, 3, 3, 1], strides=[1, 2, 2, 2, 1], padding='SAME')
x_image = tf.placeholder(tf.float32, shape=[None, None, None, None, 1])
label = tf.placeholder(tf.float32, shape=[None, None, 3])
W_conv1 = weight_variable([3, 3, 1, 1, 32])
h_conv1 = conv3d(x_image, W_conv1)
# second convolution
W_conv2 = weight_variable([3, 3, 4, 32, 64])
h_conv2 = conv3d_s1(h_conv1, W_conv2)
# third convolution path 1
W_conv3_A = weight_variable([1, 1, 1, 64, 64])
h_conv3_A = conv3d_s1(h_conv2, W_conv3_A)
# third convolution path 2
W_conv3_B = weight_variable([1, 1, 1, 64, 64])
h_conv3_B = conv3d_s1(h_conv2, W_conv3_B)
# fourth convolution path 1
W_conv4_A = weight_variable([3, 3, 1, 64, 96])
h_conv4_A = conv3d_s1(h_conv3_A, W_conv4_A)
# fourth convolution path 2
W_conv4_B = weight_variable([1, 7, 1, 64, 64])
h_conv4_B = conv3d_s1(h_conv3_B, W_conv4_B)
# fifth convolution path 2
W_conv5_B = weight_variable([1, 7, 1, 64, 64])
h_conv5_B = conv3d_s1(h_conv4_B, W_conv5_B)
# sixth convolution path 2
W_conv6_B = weight_variable([3, 3, 1, 64, 96])
h_conv6_B = conv3d_s1(h_conv5_B, W_conv6_B)
# concatenation
layer1 = tf.concat([h_conv4_A, h_conv6_B], 4)
w = tf.Variable(tf.constant(1., shape=[2, 2, 4, 1, 192]))
DeConnv1 = tf.nn.conv3d_transpose(layer1, filter=w, output_shape=tf.shape(x_image), strides=[1, 2, 2, 2, 1], padding='SAME')
final = DeConnv1
final_conv = conv3d_s1(final, weight_variable([1, 1, 1, 1, 3]))
y = tf.reshape(final_conv, [-1, 3])
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=label, logits=y))
print('x_image:', x_image)
print('DeConnv1:', DeConnv1)
print('final_conv:', final_conv)
def try_image(N, M, P, B=1):
batch_x = np.random.normal(size=[B, N, M, P, 1])
batch_y = np.ones([B, N * M * P, 3]) / 3.0
deconv_val, final_conv_val, loss = sess.run([DeConnv1, final_conv, cross_entropy],
feed_dict={x_image: batch_x, label: batch_y})
print(deconv_val.shape)
print(final_conv.shape)
print(loss)
print()
tf.global_variables_initializer().run()
try_image(32, 32, 7)
try_image(16, 16, 3)
try_image(16, 16, 3, 2)
Theoretically, it's possible. you need to set the image size of the input and label image place holder to none, and let the graph dynamically infer the image size from input data.
However, have to be careful when you define the graph. Need to use tf.shape instead of tf.get_shape(). the former dynamically infer the shape only when you session.run, the latter can get the shape when you define the graph. But when input size is set to none, the latter does not get true reshape (maybe just return None).
And to make things complicated, if you use tf.layers.conv2d or upconv2d, sometimes these high level functions do not like tf.shape, because it seems they assume the shape information are available during graph construction.
I hope I have better working example to show the points above. I'll put this answer as a placeholder and will come back and add more stuff if I get a chance.
I recently trained a convolutional neural network, and I would like to load the trained weights and use them in an external dataset, but I am having an issue.
I will post my code for training and testing, but I will say in summary that my weights and biases are being loaded in as they were previously defined. I defined my weights are random tensors and all biases as 0.1 to ensure no nodes would start off dead. When I load in this network I observe that ALL biases are 0.1 and, more troublingly, if I run my prediction twice in a row on the same data, I get two different results-- suggesting that I am somehow randomizing the weights during testing!
Anyways, here's the training code; my network and weights/biases are defined like this:
def conv3dWrap(x, W, b, strides=1,activation='yes'):
x = tf.nn.conv3d(x, W, strides=[1, strides, strides, strides, 1], padding='SAME')
x = tf.nn.bias_add(x, b)
if activation =='no' or activation == 0:
return x
return tf.nn.relu(x)
def conv_net(x, weights, biases, dropout):
#First Conv Layer
conv1 = conv3dWrap(x,weights['wConv1'],biases['bConv1'])
conv2 = conv3dWrap(conv1, weights['wConv2'], biases['bConv2'])
conv3 = conv3dWrap(conv2, weights['wConv3'], biases['bConv3'])
conv4 = conv3dWrap(conv3, weights['wConv4'], biases['bConv4'])
conv5 = conv3dWrap(conv4, weights['wConv5'], biases['bConv5'])
conv6 = conv3dWrap(conv5, weights['wConv6'], biases['bConv6'])
conv7 = conv3dWrap(conv6, weights['wConv7'], biases['bConv7'])
conv8 = conv3dWrap(conv7, weights['wConv8'], biases['bConv8'])
conv9 = conv3dWrap(conv8, weights['wConv9'], biases['bConv9'])
conv10 = conv3dWrap(conv9, weights['wConv10'], biases['bConv10'])
conv11 = conv3dWrap(conv10, weights['wConv11'], biases['bConv11'])
conv12 = conv3dWrap(conv11, weights['wConv12'], biases['bConv12'])
conv13 = conv3dWrap(conv12, weights['wConv13'], biases['bConv13'])
conv14 = conv3dWrap(conv13, weights['wConv14'], biases['bConv14'])
conv15 = conv3dWrap(conv14, weights['wConv15'], biases['bConv15'])
conv16 = conv3dWrap(conv15, weights['wConv16'], biases['bConv16'])
convOUT = conv3dWrap(conv16,weights['wConvOUT'],biases['bConvOUT'],1,0)
return convOUT
# Store layers weight & bias
weights = {
#'wConv1': tf.Variable(tf.random_normal([3, 3, 3, 1, 16],0,0.1), name='wC1'),
'wConv1': tf.Variable(tf.ones([3, 3, 3, 1, 16]), name='wC1'),
# 5x5 conv, 32 inputs, 64 outputs
'wConv2': tf.Variable(tf.random_normal([3, 3, 3, 16, 16],0,0.1), name='wC2'),
'wConv3': tf.Variable(tf.random_normal([3, 3, 3, 16, 16],0,0.1), name='wC3'),
'wConv4': tf.Variable(tf.random_normal([3, 3, 3, 16, 16],0,0.1), name='wC4'),
'wConv5': tf.Variable(tf.random_normal([3, 3, 3, 16, 16],0,0.1), name='wC5'),
'wConv6': tf.Variable(tf.random_normal([3, 3, 3, 16, 16],0,0.1), name='wC6'),
'wConv7': tf.Variable(tf.random_normal([3, 3, 3, 16, 16],0,0.1), name='wC7'),
'wConv8': tf.Variable(tf.random_normal([3, 3, 3, 16, 16],0,0.1), name='wC8'),
'wConv9': tf.Variable(tf.random_normal([3, 3, 3, 16, 16],0,0.1), name='wC9'),
'wConv10': tf.Variable(tf.random_normal([3, 3, 3, 16, 16],0,0.01), name='wC10'),
'wConv11': tf.Variable(tf.random_normal([3, 3, 3, 16, 16],0,0.01), name='wC11'),
'wConv12': tf.Variable(tf.random_normal([3, 3, 3, 16, 16],0,0.01), name='wC12'),
'wConv13': tf.Variable(tf.random_normal([3, 3, 3, 16, 16],0,0.01), name='wC13'),
'wConv14': tf.Variable(tf.random_normal([3, 3, 3, 16, 16],0,0.01), name='wC14'),
'wConv15': tf.Variable(tf.random_normal([3, 3, 3, 16, 16],0,0.01), name='wC15'),
'wConv16': tf.Variable(tf.random_normal([3, 3, 3, 16, 16],0,0.01), name='wC16'),
'wConvOUT': tf.Variable(tf.random_normal([3, 3, 3, 16, 1],0,0.01), name='wCOUT')
}
biases = {
'bConv1': tf.Variable(16*[0.01],name='bC1'),
'bConv2': tf.Variable(16*[0.01],name='bC2'),
'bConv3': tf.Variable(16*[0.01],name='bC3'),
'bConv4': tf.Variable(16*[0.01],name='bC4'),
'bConv5': tf.Variable(16*[0.01],name='bC5'),
'bConv6': tf.Variable(16*[0.01],name='bC6'),
'bConv7': tf.Variable(16*[0.01],name='bC7'),
'bConv8': tf.Variable(16*[0.01],name='bC8'),
'bConv9': tf.Variable(16*[0.01],name='bC9'),
'bConv10': tf.Variable(16*[0.01],name='bC10'),
'bConv11': tf.Variable(16*[0.01],name='bC11'),
'bConv12': tf.Variable(16*[0.01],name='bC12'),
'bConv13': tf.Variable(16*[0.01],name='bC13'),
'bConv14': tf.Variable(16*[0.01],name='bC14'),
'bConv15': tf.Variable(16*[0.01],name='bC15'),
'bConv16': tf.Variable(16*[0.01],name='bC16'),
'bConvOUT': tf.Variable([0.01],name='bCOUT')
}
And then I train the network by feeding in patches of some reference data like this:
def train_cnn(x):
epochLosses=[]
print('Beginning Training!')
pred = conv_net(x, weights, biases, KEEP_PROB)
cost = tf.reduce_mean(tf.squared_difference(pred, y))
saver = tf.train.Saver()
optimizer = tf.train.AdamOptimizer(learning_rate=LEARNING_RATE)
train_op=optimizer.minimize(cost)
#optimizer = tf.train.Ad(learning_rate=LEARNING_RATE).minimize(cost)
epochLosses=[]
valLosses=[]
with tf.Session() as sess:
print('Beginning Session!')
sess.run(tf.global_variables_initializer())
writer = tf.summary.FileWriter ( './graphs' , sess.graph)
for epoch in range(NUM_EPOCHS):
t0=time.time()
print('Epoch #:')
print(epoch)
epoch_loss=0
for pid in pidList: #get a patient's data
os.chdir(mriDir+pid)
file=glob.glob('cr*')
mri = nib.load(file[0])
mri = mri.get_data()
mri = mri-mri.mean()
#mri = mri[:63,:63,:]
mri = np.expand_dims(mri, 0)
mri = np.expand_dims(mri, -1)
os.chdir(txDir+pid)
file=glob.glob('input.n*')
tx = nib.load(file[0])
tx = tx.get_data()
#tx = tx[:63,:63,:]
tx = np.expand_dims(tx, 0)
tx = np.expand_dims(tx, -1)
tx = np.transpose(tx,(0,3,1,2,4))
mri = np.transpose(mri,(0,3,1,2,4))
os.chdir(launch)
for zSlice in range( 1 , mri.shape[2] - 1 ):
for ud in range(2):
for lr in range(2):
inPatch=mri[:,zSlice-1:zSlice+1,(ud*63):((ud+1)*63),(lr*63):((lr+1)*63),:]
testPatch=tx[:,zSlice-1:zSlice+1,(ud*63):((ud+1)*63),(lr*63):((lr+1)*63),:]
_, c = sess.run([train_op, cost], feed_dict = {x: inPatch,y: testPatch})
epoch_loss += c
save_path = saver.save(sess, "./SIMPLE_OUT/simple_test_cnn.ckpt")
# print(epoch_loss)
mean_epoch_loss=epoch_loss/len(pidList)
print('Epoch', epoch, 'completed out of',NUM_EPOCHS,'loss:',epoch_loss)
print('Mean loss for epoch:', mean_epoch_loss)
epochLosses.append(epoch_loss)
#Validation
print('Running validation for', epoch, 'will cut training if mean MSE <', VALIDATION_CUTOFF)
valLoss=0
t1=time.time()
t=t1-t0
print('EPOCH #:', epoch, 'is complete')
print('It took this much time:', t)
It goes on to do some other stuff like validation and all, but this is the important part for this question.
Then I go on to try and test the network thusly:
def conv3dWrap(x, W, b, strides=1,activation='yes'):
x = tf.nn.conv3d(x, W, strides=[1, strides, strides, strides, 1], padding='SAME')
x = tf.nn.bias_add(x, b)
if activation =='no' or activation == 0:
return x
return tf.nn.elu(x)
def conv_net(x, weights, biases, dropout):
#First Conv Layer
conv1 = conv3dWrap(x,weights['wConv1'],biases['bConv1'])
conv2 = conv3dWrap(conv1, weights['wConv2'], biases['bConv2'])
conv3 = conv3dWrap(conv2, weights['wConv3'], biases['bConv3'])
conv4 = conv3dWrap(conv3, weights['wConv4'], biases['bConv4'])
conv5 = conv3dWrap(conv4, weights['wConv5'], biases['bConv5'])
conv6 = conv3dWrap(conv5, weights['wConv6'], biases['bConv6'])
conv7 = conv3dWrap(conv6, weights['wConv7'], biases['bConv7'])
conv8 = conv3dWrap(conv7, weights['wConv8'], biases['bConv8'])
conv9 = conv3dWrap(conv8, weights['wConv9'], biases['bConv9'])
conv10 = conv3dWrap(conv9, weights['wConv10'], biases['bConv10'])
conv11 = conv3dWrap(conv10, weights['wConv11'], biases['bConv11'])
conv12 = conv3dWrap(conv11, weights['wConv12'], biases['bConv12'])
conv13 = conv3dWrap(conv12, weights['wConv13'], biases['bConv13'])
conv14 = conv3dWrap(conv13, weights['wConv14'], biases['bConv14'])
conv15 = conv3dWrap(conv14, weights['wConv15'], biases['bConv15'])
conv16 = conv3dWrap(conv15, weights['wConv16'], biases['bConv16'])
convOUT = conv3dWrap(conv16,weights['wConvOUT'],biases['bConvOUT'],1,0)
return convOUT
# Store layers weight bias
sess = tf.Session()
x = tf.placeholder(dtype=tf.float32)
new_saver = tf.train.import_meta_graph('./SIMPLE_OUT/simple_test_cnn.ckpt.meta')
sess.run(tf.global_variables_initializer())
#pred = tf.get_default_graph().get_tensor_by_name("pred:0")
weights = {
'wConv1': sess.run('wC1:0'),
'wConv2': sess.run('wC2:0'),
'wConv3': sess.run('wC3:0'),
'wConv4': sess.run('wC4:0'),
'wConv5': sess.run('wC5:0'),
'wConv6': sess.run('wC6:0'),
'wConv7': sess.run('wC7:0'),
'wConv8': sess.run('wC8:0'),
'wConv9': sess.run('wC9:0'),
'wConv10': sess.run('wC10:0'),
'wConv11': sess.run('wC11:0'),
'wConv12': sess.run('wC12:0'),
'wConv13': sess.run('wC13:0'),
'wConv14': sess.run('wC14:0'),
'wConv15': sess.run('wC15:0'),
'wConv16': sess.run('wC16:0'),
'wConvOUT' : sess.run('wCOUT:0')
}
biases = {
'bConv1': sess.run('bC1:0'),
'bConv2': sess.run('bC2:0'),
'bConv3': sess.run('bC3:0'),
'bConv4': sess.run('bC4:0'),
'bConv5': sess.run('bC5:0'),
'bConv6': sess.run('bC6:0'),
'bConv7': sess.run('bC7:0'),
'bConv8': sess.run('bC8:0'),
'bConv9': sess.run('bC9:0'),
'bConv10': sess.run('bC10:0'),
'bConv11': sess.run('bC11:0'),
'bConv12': sess.run('bC12:0'),
'bConv13': sess.run('bC13:0'),
'bConv14': sess.run('bC14:0'),
'bConv15': sess.run('bC15:0'),
'bConv16': sess.run('bC16:0'),
'bConvOUT': sess.run('bCOUT:0')
}
t0=time.time()
pred = conv_net(x, weights, biases,0)
print("Loaded Weights")
b1=sess.run(pred,feed_dict={x: mri})
b=b1[0,:,:,:,0]
a=b[80,:,:]
t=time.time()-t0
And I run into the previously mentioned issues. The biases are ALL 0.1. And, running this code twice in a row will yield two different results! I am not sure how/why I am messing this up but I've spent a long time trying to figure out and I think it's time to ask for help.
Edit: Since this would be my first guess: I'd like to confirm that the weights and biases do change during training. This is confirmed by the fact that the cost changes and, more specifically, that the L2 loss of each weight/bias changes during every backpropogration (I am not using regularization on my biases, I just did it to confirm they were changing).