I am new to Tensorflow and I have problems with combining shapes (n,) with shapes (n,1).
I have this code:
if __name__ == '__main__':
trainSetX, trainSetY = utils.load_train_set()
# create placeholders & variables
X = tf.placeholder(tf.float32, shape=(num_of_features,))
y = tf.placeholder(tf.float32, shape=(1,))
W, b = initialize_params()
# predict y
y_estim = linear_function(X, W, b)
y_pred = tf.sigmoid(y_estim)
# set the optimizer
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=y_pred)
loss_mean = tf.reduce_mean(loss)
optimizer = tf.train.GradientDescentOptimizer(learning_rate=alpha).minimize(loss_mean)
# training phase
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
for idx in range(num_of_examples):
cur_x, cur_y = trainSetX[idx], trainSetY[idx]
_, c = sess.run([optimizer, loss_mean], feed_dict={X: cur_x, y: cur_y})
I am trying to implement a stochastic gradient descent by feeding one example at the time. The problem is that it seems to feed the data in shape (num_of_features,), while I need (num_of_features,1) for the correct usage of the other functions.
For example, the code given before causes error when it comes to calculating the prediction of y with this function:
def linear_function(x, w, b):
y_est = tf.add(tf.matmul(w, x), b)
return y_est
The error is:
ValueError: Shape must be rank 2 but is rank 1 for 'MatMul' (op: 'MatMul') with input shapes: [1,3197], [3197].
I was trying to use tf.reshape with X and y to somehow solve this problem, but it caused errors in other places.
Is it possible to feed the data in feed_dict={X: cur_x, y: cur_y} in "correct" shape?
Or what is the way to properly implement this?
Thanks.
For matrix multiplications, you need to follow the rule of shapes
(a, b) * (b, c) = (a, c)
Which means you do need to reshape it since the shapes in your code are not following it. Showing what error you got after reshape would help.
Hope this gives you some hint
import tensorflow as tf
a = tf.constant([1, 2], shape=[1, 2])
b = tf.constant([7, 8], shape=[2])
print(a.shape) # => (1, 2)
print(b.shape) # => (2,)
sess = tf.Session()
# r = tf.matmul(a, b)
# print(sess.run(r)) # this gives you error
c = tf.reshape(b, [2, 1])
print(c.shape) # => (2, 1)
r = tf.matmul(a, c)
foo = tf.reshape(r, [1])
foo = sess.run(foo)
print(foo) # this gives you [23]
Related
I'm trying to implement the UNET at the keras website:
Image segmentation with a U-Net-like architecture
With only one change. use Dice loss instead of "sparse_categorical_crossentropy". However, every time I try something, I get different error. I'm coding on google colab using Tensorflow 2.7.
For example, I tried using
def DiceLoss(targets, inputs, smooth=1e-6):
#flatten label and prediction tensors
inputs = K.flatten(inputs)
targets = K.flatten(targets)
intersection = K.sum(K.dot(targets, inputs))
dice = (2*intersection + smooth) / (K.sum(targets) + K.sum(inputs) + smooth)
return 1 - dice
The eror I got:
ValueError: Shape must be rank 2 but is rank 1 for '{{node DiceLoss99/MatMul}} = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false](DiceLoss99/Reshape_1, DiceLoss99/Reshape)' with input shapes: [?], [?].
The problem is on this line:
intersection = K.sum(K.dot(targets, inputs))
I also tried this library:
!pip install git+https://github.com/qubvel/segmentation_models
# define optomizer
n_classes=3
LR = 0.0001
optim = keras.optimizers.Adam(LR)
dice_loss_sm = sm.losses.DiceLoss(class_weights=K.ones_like(n_classes))
However, I got the following error:
TypeError: Input 'y' of 'Mul' Op has type int32 that does not match type float32 of argument 'x'.
the remaining code is same as in keras.io. but I listed below for completeness :
from tensorflow.keras import layers
def get_model(img_size, num_classes):
inputs = keras.Input(shape=img_size + (3,))
### [First half of the network: downsampling inputs] ###
# Entry block
x = layers.Conv2D(32, 3, strides=2, padding="same")(inputs)
x = layers.BatchNormalization()(x)
x = layers.Activation("relu")(x)
previous_block_activation = x # Set aside residual
# Blocks 1, 2, 3 are identical apart from the feature depth.
for filters in [64, 128, 256]:
x = layers.Activation("relu")(x)
x = layers.SeparableConv2D(filters, 3, padding="same")(x)
x = layers.BatchNormalization()(x)
x = layers.Activation("relu")(x)
x = layers.SeparableConv2D(filters, 3, padding="same")(x)
x = layers.BatchNormalization()(x)
x = layers.MaxPooling2D(3, strides=2, padding="same")(x)
# Project residual
residual = layers.Conv2D(filters, 1, strides=2, padding="same")(
previous_block_activation
)
x = layers.add([x, residual]) # Add back residual
previous_block_activation = x # Set aside next residual
### [Second half of the network: upsampling inputs] ###
for filters in [256, 128, 64, 32]:
x = layers.Activation("relu")(x)
x = layers.Conv2DTranspose(filters, 3, padding="same")(x)
x = layers.BatchNormalization()(x)
x = layers.Activation("relu")(x)
x = layers.Conv2DTranspose(filters, 3, padding="same")(x)
x = layers.BatchNormalization()(x)
x = layers.UpSampling2D(2)(x)
# Project residual
residual = layers.UpSampling2D(2)(previous_block_activation)
residual = layers.Conv2D(filters, 1, padding="same")(residual)
x = layers.add([x, residual]) # Add back residual
previous_block_activation = x # Set aside next residual
# Add a per-pixel classification layer
outputs = layers.Conv2D(num_classes, 3, activation="softmax", padding="same")(x)
# Define the model
model = keras.Model(inputs, outputs)
return model
# Free up RAM in case the model definition cells were run multiple times
keras.backend.clear_session()
# Build model
model = get_model(img_size, num_classes)
model.summary()
# Configure the model for training.
# We use the "sparse" version of categorical_crossentropy
# because our target data is integers.
# notice I changed the lose the dice loss instead of sparse_categorical_crossentropy
model.compile(optimizer="rmsprop", loss="sparse_categorical_crossentropy")
callbacks = [
keras.callbacks.ModelCheckpoint("oxford_segmentation.h5", save_best_only=True)
]
# Train the model, doing validation at the end of each epoch.
epochs = 15
model.fit(train_gen, epochs=epochs, validation_data=val_gen, callbacks=callbacks)
EDIT
This detailed error message when trying the lose library at segmentation_models:
The issue on this code :
backend = kwargs['backend']
Args:
gt: ground truth 4D keras tensor (B, H, W, C) or (B, C, H, W)
pr: prediction 4D keras tensor (B, H, W, C) or (B, C, H, W)
class_weights: 1. or list of class weights, len(weights) = C
class_indexes: Optional integer or list of integers, classes to consider, if ``None`` all classes are used.
beta: f-score coefficient
smooth: value to avoid division by zero
per_image: if ``True``, metric is calculated as mean over images in batch (B),
else over whole batch
threshold: value to round predictions (use ``>`` comparison), if ``None`` prediction will not be round
Returns:
F-score in range [0, 1]
"""
Args:
gt: ground truth 4D keras tensor (B, H, W, C) or (B, C, H, W)
pr: prediction 4D keras tensor (B, H, W, C) or (B, C, H, W)
class_weights: 1. or list of class weights, len(weights) = C
class_indexes: Optional integer or list of integers, classes to consider, if ``None`` all classes are used.
beta: f-score coefficient
smooth: value to avoid division by zero
per_image: if ``True``, metric is calculated as mean over images in batch (B),
else over whole batch
threshold: value to round predictions (use ``>`` comparison), if ``None`` prediction will not be round
Returns:
F-score in range [0, 1]
"""
Args:
gt: ground truth 4D keras tensor (B, H, W, C) or (B, C, H, W)
pr: prediction 4D keras tensor (B, H, W, C) or (B, C, H, W)
class_weights: 1. or list of class weights, len(weights) = C
class_indexes: Optional integer or list of integers, classes to consider, if ``None`` all classes are used.
beta: f-score coefficient
smooth: value to avoid division by zero
per_image: if ``True``, metric is calculated as mean over images in batch (B),
else over whole batch
threshold: value to round predictions (use ``>`` comparison), if ``None`` prediction will not be round
Returns:
F-score in range [0, 1]
"""
gt, pr = gather_channels(gt, pr, indexes=class_indexes, **kwargs)
pr = round_if_needed(pr, threshold, **kwargs)
axes = get_reduce_axes(per_image, **kwargs)
# calculate score
tp = backend.sum(gt * pr, axis=axes) # the issue here
fp = backend.sum(pr, axis=axes) - tp
fn = backend.sum(gt, axis=axes) - tp
score = ((1 + beta ** 2) * tp + smooth) \
/ ((1 + beta ** 2) * tp + beta ** 2 * fn + fp + smooth)
score = average(score, per_image, class_weights, **kwargs)
return score
The code for gt,pr and axis is here:
def get_reduce_axes(per_image, **kwargs):
backend = kwargs['backend']
axes = [1, 2] if backend.image_data_format() == 'channels_last' else [2, 3]
if not per_image:
axes.insert(0, 0)
return axes
def gather_channels(*xs, indexes=None, **kwargs):
"""Slice tensors along channels axis by given indexes"""
if indexes is None:
return xs
elif isinstance(indexes, (int)):
indexes = [indexes]
xs = [_gather_channels(x, indexes=indexes, **kwargs) for x in xs]
return xs
def round_if_needed(x, threshold, **kwargs):
backend = kwargs['backend']
if threshold is not None:
x = backend.greater(x, threshold)
x = backend.cast(x, backend.floatx())
return x
You are passing 1-dimensional vectors to K.dot, while the ValueError is saying that K.dot requires arrays with 2-dimensions.
You can replace it with element-wise multiplication, i.e. intersection = K.sum(targets *inputs)
I'm doing sequence classification, I've got batch sizes of 1, 5 outcomes, and variable time steps (14 in this example). My sample weights w are the same shape as my label y:
y = tf.convert_to_tensor(np.ones(shape = (1,14,5)))
w = tf.convert_to_tensor(np.random.uniform(size = (1,14,5)))
y.shape
Out[53]: TensorShape([1, 14, 5])
w.shape
Out[54]: TensorShape([1, 14, 5])
When I try to run this through the loss function, I get the following error:
loss_object = tf.keras.losses.BinaryCrossentropy(from_logits=False)
loss_object(y_true = y,
y_pred = y,
sample_weight = w)
InvalidArgumentError: Can not squeeze dim[2], expected a dimension of 1, got 5 [Op:Squeeze]
what's going on? It should be a straightforward multiplication of a loss matrix (pre-reduction) with the weights. How to fix?
Super simple fix! Tensorflow squeezes the last dimension of the sample weights because they are supposed to be applied per sample, therefore, all you need to do is add one dimension to your weight matrix along the last axis:
y = tf.convert_to_tensor(np.ones(shape = (1,14,5)))
w = tf.convert_to_tensor(np.random.uniform(size = (1,14,5,1))) # Change made here
loss_object = tf.keras.losses.BinaryCrossentropy(from_logits=False)
loss_object(y_true = y,
y_pred = y,
sample_weight = w)
You can also just change the shape of the weights matrix after creation:
w = tf.expand_dims(w, axis=-1)
I am following tutorial on simple audio recognition and currently editing the label_wav.py. In original case we input wave file and the graph predicts the label (in between it calculates spectrum, mfcc's inside the graph). Now i am looking to input mfcc's directly rather than inputting the wave file. Run the graph by inputting the mfcc tensor.
# mfccs: Tensor("strided_slice:0", shape=(1, 98, 40), dtype=float32)
mfcc_input_layer_name = 'Reshape:0'
with tf.Session() as sess:
predictions, = sess.run(softmax_tensor, {mfcc_input_layer_name: mfcc})
After a bit of googling, i found some discussion in git and created a session_handle.
# mfccs: Tensor("strided_slice:0", shape=(1, 98, 40), dtype=float32)
mfcc_input_layer_name = 'Reshape:0'
with tf.Session() as sess:
h = tf.get_session_handle(mfccs)
h = sess.run(h)
predictions, = sess.run(softmax_tensor, {mfcc_input_layer_name: h})
The code is working as expected but I am wondering if there could be a better way of dealing with the tensor rather than creating the handle and then passing it?
I suppose you want to replace an intermediate Tensor with a value by feed_dict. If you have a Tensor object, you can replace it by feed_dict as the following
a = tf.constant(3, name="a")
b = tf.constant(4, name="b")
c = tf.add(a, b, name="c")
d = c * 3
with tf.Session() as sess:
print(sess.run(d))
print(sess.run(d, feed_dict={c: 2}))
Even though you don't have the Tensor object, you can get it by get_tensor_by_name
a = tf.constant(3, name="a")
b = tf.constant(4, name="b")
c = tf.add(a, b, name="c")
d = c * 3
with tf.Session() as sess:
c_tensor = tf.get_default_graph().get_tensor_by_name("c:0")
print(sess.run(d, feed_dict={c_tensor: 2}))
I just browsed through Stack Overflow and other forums but couldn't find anything helpful for my problem. But it seems related to this question.
I currently have a trained model of Tensorflow (128 inputs and 11 outputs) which I saved, following the MNIST tutorial by Tensorflow.
It seemed to be successful and I have a model in this folder now with the 3 files (.meta, .ckpt.data and .index). However, I want to restore it and use it for predictions:
#encoding[0] => numpy ndarray (128, ) # anyway a list with only one entry
#unknowndata = np.array(encoding[0])[None]
unknowndata = np.expand_dims(encoding[0], axis=0)
print(unknowndata.shape) # Output (1, 128)
# Restore pre-trained tf model
with tf.Session() as sess:
#saver.restore(sess, "models/model_1.ckpt")
saver = tf.train.import_meta_graph('models/model_1.ckpt.meta')
saver.restore(sess,tf.train.latest_checkpoint('models/./'))
y = tf.get_collection('final tensor') # tf.nn.softmax(tf.matmul(y2, W3) + b3)
X = tf.get_collection('input') # tf.placeholder(tf.float32, [None, 128])
# W1 = tf.get_collection('vars')[0]
# b1 = tf.get_collection('vars')[1]
# W2 = tf.get_collection('vars')[2]
# b2 = tf.get_collection('vars')[3]
# W3 = tf.get_collection('vars')[4]
# b3 = tf.get_collection('vars')[5]
# y1 = tf.nn.relu(tf.matmul(X, W1) + b1)
# y2 = tf.nn.relu(tf.matmul(y1, W2) + b2)
# yLog = tf.matmul(y2, W3) + b3
# y = tf.nn.softmax(yLog)
prediction = tf.argmax(y, 1)
print(sess.run(prediction, feed_dict={i: d for i,d in zip(X, unknowndata.T)}))
# also had sess.run(prediction, feed_dict={X: unknowndata.T}) and also not transposed, still errors
# Output: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] # one should be 1 obviously with a specific percentage
There I only run in problems ...
ValueError: Cannot feed value of shape (1,) for Tensor 'x:0', which has shape '(?, 128)'
Altough I print the shape of the 'unknowndata' and it matches the (1, 128).
I also tried it with
sess.run(prediction, feed_dict={X: unknownData})) # with transposed etc. but nothing worked for me there I got the other error
TypeError: unhashable type: 'list'
I only want some predictions of this beautiful Tensorflow trained model.
I figured out the problem!
First I need to restore all of the values (weights and biases and matmul them seperately).
Second I need to create the same input as in the trained model, in my case:
X = tf.placeholder(tf.float32, [None, 128])
and then just call the prediction:
sess.run(prediction, feed_dict={X: unknownData})
But I do not get any percentage distribution but I expect that due to the softmax function. Does anybody know how to access those?
The prediction tensor is obtained by an argmax on y. Instead of returning only prediction, you can add y to your output feed when you execute sess.run.
output_feed = [prediction, y]
preds, probs = sess.run(output_feed, print(sess.run(prediction, feed_dict={i: d for i,d in zip(X, unknowndata.T)}))
preds will have the predictions of the model andprobs will have the probability scores.
First when you save, you have to add to the collection the placeholders you need tf.add_to_collection('i', i) and then retrieve them and pass them the feed_dict.
In your example is "i":
i = tf.get_collection('i')[0]
#sess.run(prediction, feed_dict={i: d for i,d in zip(X, unknowndata.T)})
I have the following function:
def forward_propagation(self, x):
# The total number of time steps
T = len(x)
# During forward propagation we save all hidden states in s because need them later.
# We add one additional element for the initial hidden, which we set to 0
s = tf.zeros([T+1, self.hidden_dim])
# The outputs at each time step. Again, we save them for later.
o = tf.zeros([T, self.word_dim])
a = tf.placeholder(tf.float32)
b = tf.placeholder(tf.float32)
c = tf.placeholder(tf.float32)
s_t = tf.nn.tanh(a + tf.reduce_sum(tf.multiply(b, c)))
o_t = tf.nn.softmax(tf.reduce_sum(tf.multiply(a, b)))
# For each time step...
with tf.Session() as sess:
s = sess.run(s)
o = sess.run(o)
for t in range(T):
# Note that we are indexing U by x[t]. This is the same as multiplying U with a one-hot vector.
s[t] = sess.run(s_t, feed_dict={a: self.U[:, x[t]], b: self.W, c: s[t-1]})
o[t] = sess.run(o_t, feed_dict={a: self.V, b: s[t]})
return [o, s]
self.U, self.V, and self.W are numpy arrays. I try to get softmax on
o_t = tf.nn.softmax(tf.reduce_sum(tf.multiply(a, b)))
graph, and it gives me error on this line:
o[t] = sess.run(o_t, feed_dict={a: self.V, b: s[t]})
The error is:
InvalidArgumentError (see above for traceback): Expected begin[0] == 0
(got -1) and size[0] == 0 (got 1) when input.dim_size(0) == 0
[[Node: Slice = Slice[Index=DT_INT32, T=DT_INT32,
_device="/job:localhost/replica:0/task:0/cpu:0"](Shape_1, Slice/begin, Slice/size)]]
How I am supposed to get softmax in tensorflow?
The problem arises because you call tf.reduce_sum on the argument of tf.nn.softmax. As a result, the softmax function fails because a scalar is not a valid input argument. Did you mean to use tf.matmul instead of the combination of tf.reduce_sum and tf.multiply?
Edit: Tensorflow does not provide an equivalent of np.dot out of the box. If you want to compute the dot product of a matrix and a vector, you need to sum over indices explicitly:
# equivalent to np.dot(a, b) if a.ndim == 2 and b.ndim == 1
c = tf.reduce_sum(a * b, axis=1)