I am training an autoencoder by giving 2 placeholders that store the following:
x1 = [x1]
X = [x1,x2,x3...xn]
It holds that:
y1 = W*x1 + b_encoding1
Therefore, I have a variable named b_encoder1 (the b)
(When I print it I get: <tf.Variable 'b_encoder1:0' shape=(10,) dtype=float32_ref>)
But it also holds that:
Y = W*X + b_encoding1
The size of the second b_encoding1 has to be (10,n) intead of (10,). How can I augment it and pass it in tensorflow?
Y = tf.compat.v1.nn.xw_plus_b(X, W1, b_encoder1, name='Y')
The whole code looks like this:
x1 = tf.compat.v1.placeholder( tf.float32, [None,input_shape], name = 'x1')
X = tf.compat.v1.placeholder( tf.float32, [None,input_shape,sp], name = 'X')
W1 = tf.Variable(tf.initializers.GlorotUniform()(shape=[input_shape,code_length]),name='W1')
b_encoder1 = tf.compat.v1.get_variable(name='b_encoder1',shape=[code_length],initializer=tf.compat.v1.initializers.zeros(), use_resource=False)
K = tf.Variable(tf.initializers.GlorotUniform()(shape=[code_length,code_length]),name='K')
b_decoder1 = tf.compat.v1.get_variable(name='b_decoder1',shape=[input_shape],initializer=tf.compat.v1.initializers.zeros(), use_resource=False)
y1 = tf.compat.v1.nn.xw_plus_b(x1, W1, b_encoder1, name='y1')
Y = tf.compat.v1.nn.xw_plus_b(X, W1, b_encoder1, name='Y')
I also declare the loss function and so on and then train with:
with tf.compat.v1.Session() as sess:
sess.run(tf.compat.v1.global_variables_initializer())
for epoch_i in range(epochs):
for batch_i in range(number_of_batches):
batch_data = getBatch(shuffled_data, batch_i, batch_size)
sess.run(optimizer, feed_dict={x1: batch_data[:,:,0], X: batch_data})
train_loss = sess.run(loss, feed_dict={x1: aug_data[:,:,0], X: aug_data})
print(epoch_i, train_loss)
You can consider X as a batch of x. X can take in an arbitrary number of samples:
import tensorflow as tf
import numpy as np
X = tf.placeholder(shape=(None, 100), dtype=tf.float32)
W = tf.get_variable('kernel', [100,10])
b = tf.get_variable('bias',[10])
Y = tf.nn.xw_plus_b(X, W,b, name='Y')
with tf.Session() as sess:
sess.run(tf.global_variables_initializer()) # tf version < 1.13
out = sess.run(Y, {X: np.random.rand(128, 100)}) # here n=128
Note that dimension of bias b is still 10-D regardless value of n.
Please try:
b_encoding1 = tf.expand_dims(b_encoding1, axis = 1)
Related
Im doing a tutorial on tensorflow and I have following task:
I need to create 10 logits in tensorflow with 2 hidden layers and calculate the loss of them using the functions
tf.nn.softmax()
tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels)
My code looks like this:
import tensorflow as tf, numpy as np
trainx = np.array(trainx) / 300.0
trainy = np.array(trainy)
testx = np.array(testx) / 300.0
testy = np.array(testy)
print ('train x shape is {}'.format(trainx.shape))
print ('train y shape is {}'.format(trainy.shape))
print ('test x shape is {}'.format(testx.shape))
print ('tesy y shape is {}'.format(testy.shape))
learning_rate = 0.001
def get_train_batch():
indices = np.random.randint(low=0, high=60000, size=[64])
return trainx[indices], trainy[indices]
g = tf.Graph()
with g.as_default():
x = tf.placeholder(float32, shape=[None, 784])
y = tf.placeholder(float32, shape=[None, 784])
def fc(tensor, outdim, name):
w = tf.get_variable(name + "w", shape=[1, outdim], dtype=float32)
b = tf.get_variable(name + "b", shape=[], dtype=float32, initializer=tf.constant_initializer(0.0) to tf.get_variable)
return tf.add(tf.linalg.matmul(tensor, w), b)
first = fc(trainx, trainx.shape, "first")
first = tf.nn.relu(first)
second = fc(first, trainx.shape, "second")
second = tf.nn.relu(second)
logits = []
names = [some names]
logits.append(tf.log_sigmoid(first))
logits.append(tf.log_sigmoid(second))
for i in range(10):
logits.append(fc(trainx, trainx.shape, names[0]))
tf.nn.softmax() // how do I use this paragraph to get the loss
tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels)
loss = ? // how do i calculate the loss here?
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
minimize_op = optimizer.minimize(loss, var_list=tf.trainable_variables())
sess = tf.Session()
sess.run(tf.global_variables_initializer())
for epoch in range(50000):
randomX, randomY = get_train_batch()
sess.run(minimize_op, feed_dict={x: randomX, y: randomY})
if epoch % 50 == 0:
trainloss = sess.run(loss, feed_dict={x: testx, y:testy})
print("Current loss: ", trainloss)
My questions are:
How do I use the mentioned functions correctly to calculate the loss
Where do I get the labels from?
I am using an LSTMBlockFusedCell for a network I'm working on.
When fed a single input of size [6, 3169] and an output of [-1, 3169] (casts to input size), it operates properly and predicts. The problem comes when I try and batch those same inputs. With a batch of 100, the input reshapes fine, but the output broadcasts into [600, 3169]. I have tried setting the placeholder specs exactly to the specified input length, but the same error happened. I'm pretty confident that my data is in the correct shape. I run the batch generator and print the output sizes afterward.
Here's my network:
def rnn(x, weight, bias, n_input, vocab):
x = tf.reshape(x, [-1, n_input, vocab])
rnn_cell = tf.contrib.rnn.LSTMBlockFusedCell(n_hidden)
outputs, states = rnn_cell(x, dtype=tf.float32)
return tf.matmul(outputs[-1], weight['output']) + bias['output']
my batch generator:
def new_data(dat, dic, n_steps, batch_size=100):
x = np.zeros(shape=(batch_size, n_steps, len(dic)))
y = np.zeros(shape=(batch_size, n_steps, len(dic)))
j = 0
x_dat = np.zeros(shape=(n_steps, len(dic)))
for sen in dat:
if len(sen) - 1 > n_steps:
for i, word in enumerate(sen[0:n_steps]):
x_dat[i] = one_hot(word, dic)
y_dat = one_hot(sen[n_steps], dic)
x[j % batch_size] = x_dat
y[j % batch_size] = y_dat
if j % batch_size == 0:
yield x,y
x = np.zeros(shape=(batch_size, n_steps, len(dic)))
y = np.zeros(shape=(batch_size, n_steps, len(dic)))
j += 1
and my setup:
X = tf.placeholder(tf.float32)
Y = tf.placeholder(tf.float32)
weights = {
"output" : tf.Variable(tf.random_normal([n_hidden, vocab_size]),name="weight_output")
}
bias = {
"output" : tf.Variable(tf.random_normal([vocab_size]), name="bias_output")
}
pred = rnn(X, weights, bias, n_input, vocab)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=pred, labels=Y))
with tf.Session() as sess:
sess.run(init)
step = 0
for epoch in range(n_epochs):
for x,y in new_data(dat, dic, n_steps):
_ , c = sess.run([optimizer, cost], feed_dict={X: x ,Y: y})
I'm trying to train a neural network to predict the sum of two numbers. But I don't understand what's wrong with my model. Model consists of 2 inputs, 2 hidden and 1 output layers. Every 1000 iteration I print test execution, but the result is getting smaller and smaller.
import numpy as np
import tensorflow as tf
input_size = 2
hidden_size = 3
out_size = 1
def generate_test_data():
inp = 0.5*np.random.rand(10, 2)
oup = np.zeros((10, 1))
for idx, val in enumerate(inp):
oup[idx] = np.array([val[0] + val[1]])
return inp, oup
def create_network():
x = tf.placeholder(tf.float32, [None, input_size])
w01 = tf.Variable(tf.truncated_normal([input_size, hidden_size], stddev=0.1))
y1 = tf.sigmoid(tf.matmul(tf.sigmoid(x), w01))
w12 = tf.Variable(tf.truncated_normal([hidden_size, out_size], stddev=0.1))
y2 = tf.sigmoid(tf.matmul(y1, w12))
y_ = tf.placeholder(tf.float32, [None, out_size])
return x, y_, y2
def train(x, y_, y2):
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y2)
)
train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()
# Train
for i in range(100000):
batch_xs, batch_ys = generate_test_data()
sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})
# Test
if i % 1000 == 0:
out_batch = sess.run(y2, {x: batch_xs})
inx = 0
print(batch_xs[inx][0], " + ", batch_xs[inx][1], " = ", out_batch[inx][0])
(x, y_, y2) = create_network()
train(x, y_, y2)
Output every 1000 iteration:
0.37301352864927173 + 0.28949461772342683 = 0.49111518
0.050899466843458474 + 0.006174158992116541 = 0.0025260744
0.3974852369427063 + 0.22402098418952499 = 0.00090828544
0.15735921047969498 + 0.39645077887600294 = 0.0005903727
0.23560825884336228 + 0.29010766384718145 = 0.0004317883
0.4250063393420791 + 0.24181166029062096 = 0.00031525563
= smaller and smaller
Cross-entropy loss is used for classification problems, while your task is clearly a regression. The computed cross_entropy value doesn't make sense, hence the result.
Change your loss to:
cross_entropy = tf.reduce_mean(
tf.nn.l2_loss(y_ - y2)
)
... and you'll see much more sensible results.
Maxim, thanks a lot. Now it's work.
import numpy as np
import tensorflow as tf
input_size = 2
hidden_size = 3
out_size = 1
def generate_test_data():
inp = 0.5*np.random.rand(10, 2)
oup = np.zeros((10, 1))
for idx, val in enumerate(inp):
oup[idx] = np.array([val[0] + val[1]])
return inp, oup
def create_network():
x = tf.placeholder(tf.float32, [None, input_size])
w01 = tf.Variable(tf.truncated_normal([input_size, hidden_size], stddev=0.1))
y1 = tf.matmul(x, w01)
w12 = tf.Variable(tf.truncated_normal([hidden_size, out_size], stddev=0.1))
y2 = tf.matmul(y1, w12)
y_ = tf.placeholder(tf.float32, [None, out_size])
return x, y_, y2
def train(x, y_, y2):
cross_entropy = tf.reduce_mean(
tf.nn.l2_loss(y_ - y2)
)
train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()
# Train
for i in range(100000):
batch_xs, batch_ys = generate_test_data()
sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})
# Test
if i % 2000 == 0:
out_batch = sess.run(y2, {x: batch_xs})
inx = 0
print(batch_xs[inx][0], " + ", batch_xs[inx][1], " = ", out_batch[inx][0], "|", batch_xs[inx][0] + batch_xs[inx][1])
(x, y_, y2) = create_network()
train(x, y_, y2)
If you consider predicting each digit to be a classification problem where you predict a value in "0123456789 ", you can use cross-entropy as your loss. For reference, see the Keras - Addition RNN Example.
But like Maxim said, it shouldn't be used for a regression problem.
I'm trying to use Numpy arrays within a graph, feeding in the data using a Dataset.
I've read through this, but can't quite make sense of how I should feed placeholder arrays within a Dataset.
If we take a simple example, I start with:
A = np.arange(4)
B = np.arange(10, 14)
a = tf.placeholder(tf.float32, [None])
b = tf.placeholder(tf.float32, [None])
c = tf.add(a, b)
with tf.Session() as sess:
for i in range(10):
x = sess.run(c, feed_dict={a: A, b:B})
print(i, x)
Then I attempt to modify it to use a Dataset as follows:
A = np.arange(4)
B = np.arange(10, 14)
a = tf.placeholder(tf.int32, A.shape)
b = tf.placeholder(tf.int32, B.shape)
c = tf.add(a, b)
dataset = tf.data.Dataset.from_tensors((a, b))
iterator = dataset.make_initializable_iterator()
with tf.Session() as sess3:
sess3.run(tf.global_variables_initializer())
sess3.run(iterator.initializer, feed_dict={a: A, b: B})
for i in range(10):
x = sess3.run(c)
print(i, x)
If I run this I get 'InvalidArgumentError: You must feed a value for placeholder tensor ...'
The code until the for loop mimics the example here, but I don't get how I can then employ the placeholders a & b without supplying a feed_dict to every call to sess3.run(c) [which would be expensive]. I suspect I have to somehow use the iterator, but I don't understand how.
Update
It appears I oversimplified too much when picking the example. What I am really trying to do is use Datasets when training a neural network, or similar.
For a more sensible question, how would I go about using Datasets to feed placeholders in the below (though imagine X and Y_true are much longer...). The documentation takes me to the point where the loop starts and then I'm not sure.
X = np.arange(8.).reshape(4, 2)
Y_true = np.array([0, 0, 1, 1])
x = tf.placeholder(tf.float32, [None, 2], name='x')
y_true = tf.placeholder(tf.float32, [None], name='y_true')
w = tf.Variable(np.random.randn(2, 1), name='w', dtype=tf.float32)
y = tf.squeeze(tf.matmul(x, w), name='y')
loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(
labels=y_true, logits=y),
name='x_entropy')
# set optimiser
optimiser = tf.train.AdamOptimizer().minimize(loss)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for i in range(100):
_, loss_out = sess.run([optimiser, loss], feed_dict={x: X, y_true:Y_true})
print(i, loss_out)
Trying the following only gets me a InvalidArgumentError
X = np.arange(8.).reshape(4, 2)
Y_true = np.array([0, 0, 1, 1])
x = tf.placeholder(tf.float32, [None, 2], name='x')
y_true = tf.placeholder(tf.float32, [None], name='y_true')
dataset = tf.data.Dataset.from_tensor_slices((x, y_true))
iterator = dataset.make_initializable_iterator()
w = tf.Variable(np.random.randn(2, 1), name='w', dtype=tf.float32)
y = tf.squeeze(tf.matmul(x, w), name='y')
loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(
labels=y_true, logits=y),
name='x_entropy')
# set optimiser
optimiser = tf.train.AdamOptimizer().minimize(loss)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(iterator.initializer, feed_dict={x: X,
y_true: Y_true})
for i in range(100):
_, loss_out = sess.run([optimiser, loss])
print(i, loss_out)
Use iterator.get_next() to get elements from Dataset like:
next_element = iterator.get_next()
than initialize the iterator
sess.run(iterator.initializer, feed_dict={a:A, b:B})
and at least get the values from Dataset
value = sess.run(next_element)
EDIT:
The code above just return the elements from Dataset. The Dataset API is intended to serve features and labels for a input_fn, therefore all additional computations for preprocessing should be performed within the Dataset API. If you want to add elements, you should define a function that is applied to the elements, like:
def add_fn(exp1, exp2):
return tf.add(exp1, exp2)
and than you can map these function to your Dataset:
dataset = dataset.map(add_fn)
Complete code example:
A = np.arange(4)
B = np.arange(10, 14)
a = tf.placeholder(tf.int32, A.shape)
b = tf.placeholder(tf.int32, B.shape)
#c = tf.add(a, b)
def add_fn(exp1, exp2):
return tf.add(exp1, exp2)
dataset = tf.data.Dataset.from_tensors((a, b))
dataset = dataset.map(add_fn)
iterator = dataset.make_initializable_iterator()
next_element = iterator.get_next()
with tf.Session() as sess:
sess.run(iterator.initializer, feed_dict={a: A, b: B})
# just one element at dataset
x = sess.run(next_element)
print(x)
The problem in your more complicated example is that you use the same tf.placeholder() nodes as the input to the Dataset.from_tensor_slices() (which is correct) and the network itself (which causes the InvalidArgumentError. Instead, as J.E.K points out in their answer, you should use iterator.get_next() as the input to your network, as follows (note that there are a couple of other fixes I added to make the code run as-is):
X = np.arange(8.).reshape(4, 2)
Y_true = np.array([0, 0, 1, 1])
x = tf.placeholder(tf.float32, [None, 2], name='x')
y_true = tf.placeholder(tf.float32, [None], name='y_true')
dataset = tf.data.Dataset.from_tensor_slices((x, y_true))
# You will need to repeat the input (which has 4 elements) to be able to take
# 100 steps.
dataset = dataset.repeat()
iterator = dataset.make_initializable_iterator()
# Use `iterator.get_next()` to create tensors that will consume values from the
# dataset.
x_next, y_true_next = iterator.get_next()
w = tf.Variable(np.random.randn(2, 1), name='w', dtype=tf.float32)
# The `x_next` tensor is a vector (i.e. a row of `X`), so you will need to
# convert it to a matrix or apply batching in the dataset to make it work with
# `tf.matmul()`
x_next = tf.expand_dims(x_next, 0)
y = tf.squeeze(tf.matmul(x_next, w), name='y') # Use `x_next` here.
loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
labels=y_true_next, logits=y), # Use `y_true_next` here.
name='x_entropy')
# set optimiser
optimiser = tf.train.AdamOptimizer().minimize(loss)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(iterator.initializer, feed_dict={x: X,
y_true: Y_true})
for i in range(100):
_, loss_out = sess.run([optimiser, loss])
print(i, loss_out)
I am a newbie on tensorflow and learning from very beginning. I tried to read csv file and decode using tf.decode_csv and I am trying to understand batch and graph through examples.
My example csv is 8 datas and 1 logistic shape, separated by comma.
This is first four rows of my file.
-0.294118,0.487437,0.180328,-0.292929,0,0.00149028,-0.53117,-0.0333333,0
-0.882353,-0.145729,0.0819672,-0.414141,0,-0.207153,-0.766866,-0.666667,1
-0.0588235,0.839196,0.0491803,0,0,-0.305514,-0.492741,-0.633333,0
-0.882353,-0.105528,0.0819672,-0.535354,-0.777778,-0.162444,-0.923997,0,1
My code is,
import tensorflow as tf
filename_queue = tf.train.string_input_producer(['data-03-diabetes.csv'],
shuffle=False,
name='filename_queue')
reader = tf.TextLineReader()
key, value = reader.read(filename_queue)
record_defaults = [[0.],[0.],[0.],[0.],[0.],[0.],[0.],[0.],[0.]]
xy = tf.decode_csv(value, record_defaults=record_defaults, field_delim=',')
train_x_batch, train_y_batch = tf.train.batch([xy[0:-1], xy[-1:]],
batch_size=10)
X = tf.placeholder(tf.float32, shape=[None, 8])
Y = tf.placeholder(tf.float32, shape=[None, 1])
W = tf.Variable(tf.random_normal([8,1]), name='weight')
b = tf.Variable(tf.random_normal([1]), name='bias')
hypothesis = tf.sigmoid(tf.matmul(X, W) + b)
cost = -tf.reduce_mean(Y * tf.log(hypothesis) + (1 - Y) * tf.log(1 - hypothesis))
train = tf.train.GradientDescentOptimizer(learning_rate=0.1).minimize(cost)
predicted = tf.cast(hypothesis > 0.5, dtype = tf.float32)
accuracy = tf.reduce_mean(tf.cast(tf.equal(predicted, Y), dtype=tf.float32))
with tf.Session() as sess:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
sess.run(tf.global_variables_initializer())
feed = {X: train_x_batch, Y: train_y_batch}
for step in range(10001):
x_batch, y_batch = sess.run([train_x_batch, train_y_batch])
if step % 200 == 0:
print(step, sess.run(cost, feed_dict=feed))
h, c, a = sess.run([hypothesis, predicted, accuracy], feed_dict=feed)
print("\nHypothesis", h, "\nCorrect (Y):", c, "\nAccuracy", a)
and then I got this error code,
TypeError: The value of a feed cannot be a tf.Tensor object. Acceptable feed values include Python scalars, strings, lists, numpy ndarrays, or TensorHandles.
Does it mean that I declared wrong placeholder and Variable??
Thanks in advance!