For example, we have 64*100 input data which will be send to the tensor flow graph, and it will generate 64*(n_hidden nodes) output before putting into softmax or whatever loss function. We put 1*100 into the same graph, the result should be the the first row of previous output, but results are not. I use the tensor flow example on Mnist to test the comparison.
'''
A Multilayer Perceptron implementation example using TensorFlow library.
This example is using the MNIST database of handwritten digits
(http://yann.lecun.com/exdb/mnist/)
Author: Aymeric Damien
Project: https://github.com/aymericdamien/TensorFlow-Examples/
'''
from __future__ import print_function
import numpy as np
# Import MNIST data
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
import tensorflow as tf
# Parameters
learning_rate = 0.001
training_epochs = 15
batch_size = 100
display_step = 1
# Network Parameters
n_hidden_1 = 256 # 1st layer number of features
n_hidden_2 = 256 # 2nd layer number of features
n_input = 784 # MNIST data input (img shape: 28*28)
n_classes = 10 # MNIST total classes (0-9 digits)
# tf Graph input
x = tf.placeholder("float", [None, n_input])
y = tf.placeholder("float", [None, n_classes])
# Create model
def multilayer_perceptron(x, weights, biases):
# Hidden layer with RELU activation
layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])
layer_1 = tf.nn.relu(layer_1)
# Hidden layer with RELU activation
layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2'])
layer_2 = tf.nn.relu(layer_2)
# Output layer with linear activation
out_layer = tf.matmul(layer_2, weights['out']) + biases['out']
return out_layer
# Store layers weight & bias
weights = {
'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1]), name ='layer1'),
'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2]), name = 'layer2'),
'out': tf.Variable(tf.random_normal([n_hidden_2, n_classes]), name = 'layer3')
}
biases = {
'b1': tf.Variable(tf.random_normal([n_hidden_1]), name = 'layer1_b'),
'b2': tf.Variable(tf.random_normal([n_hidden_2]), name = 'layer2_b'),
'out': tf.Variable(tf.random_normal([n_classes]), name = 'layer3_b')
}
# Construct model
pred = multilayer_perceptron(x, weights, biases)
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y))
#optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
var = tf.all_variables()
trainer = tf.train.AdamOptimizer(learning_rate=learning_rate)
grads = trainer.compute_gradients(cost, var)
update = trainer.apply_gradients(grads)
# Initializing the variables
init = tf.initialize_all_variables()
# Launch the graph
with tf.Session() as sess:
sess.run(init)
# Training cycle
for epoch in range(training_epochs):
avg_cost = 0.
total_batch = int(mnist.train.num_examples/batch_size)
# Loop over all batches
for i in range(total_batch):
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Run optimization op (backprop) and cost op (to get loss value)
#_, c = sess.run([optimizer, cost], feed_dict={x: batch_x, y: batch_y})
#c, v,grad, Pred, bi = sess.run([cost, var,grads, pred, biases], feed_dict={x: batch_x, y: batch_y})
Pred_2 = sess.run(pred, feed_dict={x: batch_x, y: batch_y})
Pred_1 = sess.run(pred , feed_dict={x: batch_x[0:1,:], y: batch_y[0:1]})
print(Pred_2[0] == Pred_1)
# Compute average loss
avg_cost += c / total_batch
# Display logs per epoch step
if epoch % display_step == 0:
print("Epoch:", '%04d' % (epoch+1), "cost=", \
"{:.9f}".format(avg_cost))
# print(len(v))
# g1 = np.array(grad[0])
# g2 = np.array(grad[1])
# g3 = np.array(grad[2])
# g4 = np.array(grad[3])
# g5 = np.array(grad[4])
# g6 = np.array(grad[5])
# print(g1.shape)
# print(g2.shape)
# print(g3.shape)
# print(g4.shape)
# print(g5.shape)
# print(g6.shape)
# print(g6[0,:])
# print(g6[1,:])
# print(bi['out'])
#print(type(updating))
print("Optimization Finished!")
# Test model
correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
# Calculate accuracy
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
print("Accuracy:", accuracy.eval({x: mnist.test.images, y: mnist.test.labels}))
print(Pred_2[0] == Pred_1) Should be the same, but they are not. It is strange.
The gradient descent path should be different if your weight and bias initialization is random and the gradients each time are different, it might take a path towards a different minimum.
Related
I am starting to work again on tensorflow. I was relaunching some codes I did a few years ago, it's not working though.
Old version
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
import tensorflow as tf
# Parameters
learning_rate = 0.001
training_epochs = 20
batch_size = 128 # Decrease batch size if you don't have enough memory
display_step = 1
n_input = 784 # MNIST data input (img shape: 28*28)
n_classes = 10 # MNIST total classes (0-9 digits)
n_hidden_layer = 256 # layer number of features
# Store layers weight & bias
weights = {
'hidden_layer': tf.Variable(tf.random_normal([n_input, n_hidden_layer])),
'out': tf.Variable(tf.random_normal([n_hidden_layer, n_classes]))
}
biases = {
'hidden_layer': tf.Variable(tf.random_normal([n_hidden_layer])),
'out': tf.Variable(tf.random_normal([n_classes]))
}
# tf Graph input
x = tf.placeholder("float", [None, 28, 28, 1])
y = tf.placeholder("float", [None, n_classes])
x_flat = tf.reshape(x, [-1, n_input])
# Hidden layer with RELU activation
layer_1 = tf.add(tf.matmul(x_flat, weights['hidden_layer']),\
biases['hidden_layer'])
layer_1 = tf.nn.relu(layer_1)
# Output layer with linear activation
logits = tf.add(tf.matmul(layer_1, weights['out']), biases['out'])
# Define loss and optimizer
cost = tf.reduce_mean(\
tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=y))
optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\
.minimize(cost)
# Initializing the variables
init = tf.global_variables_initializer()
# Launch the graph
with tf.Session() as sess:
sess.run(init)
# Training cycle
for epoch in range(training_epochs):
total_batch = int(mnist.train.num_examples/batch_size)
# Loop over all batches
for i in range(total_batch):
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Run optimization op (backprop) and cost op (to get loss value)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
From what I have understood the cause comes from the "read_data_sets" and I should use "tf.data". The problem with "tf.data" is that I cannot use that anymore:
mnist.train.num_examples
mnist.train.next_batch
And the data is not one encoded.
I have tried something like that:
import tensorflow_datasets as tfds
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
# Mandatory: to launch
#tf.enable_eager_execution()
mnist_data, info = tfds.load("mnist", with_info=True, as_supervised=True)
mnist_train, mnist_test = mnist_data["train"], mnist_data["test"]
And, mnist_train.batch instead of mnist.train.next_batch
# Launch the graph
with tf.Session() as sess:
sess.run(init)
# Training cycle
for epoch in range(training_epochs):
total_batch = int(info.splits["train"].num_examples/batch_size)
print(total_batch)
# Loop over all batches
for i in range(total_batch):
batch_x, batch_y = mnist_train.batch(batch_size)
# Run optimization op (backprop) and cost op (to get loss value)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
With the error:
RuntimeError: dataset.__iter__() is only supported when eager execution is enabled.
And if I do :
tf.enable_eager_execution()
I cannot use
tf.placeholder()
New version
import tensorflow_datasets as tfds
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
# Mandatory: to launch
#tf.enable_eager_execution()
mnist_data, info = tfds.load("mnist", with_info=True, as_supervised=True)
mnist_train, mnist_test = mnist_data["train"], mnist_data["test"]
import tensorflow as tf
# Parameters
learning_rate = 0.001
training_epochs = 20
batch_size = 128 # Decrease batch size if you don't have enough memory
display_step = 1
n_input = 784 # MNIST data input (img shape: 28*28)
n_classes = 10 # MNIST total classes (0-9 digits)
n_hidden_layer = 256 # layer number of features
# Store layers weight & bias
weights = {
'hidden_layer': tf.Variable(tf.random_normal([n_input, n_hidden_layer])),
'out': tf.Variable(tf.random_normal([n_hidden_layer, n_classes]))
}
biases = {
'hidden_layer': tf.Variable(tf.random_normal([n_hidden_layer])),
'out': tf.Variable(tf.random_normal([n_classes]))
}
# tf Graph input
x = tf.placeholder("float", [None, 28, 28, 1])
y = tf.placeholder("float", [None, n_classes])
x_flat = tf.reshape(x, [-1, n_input])
# Hidden layer with RELU activation
layer_1 = tf.add(tf.matmul(x_flat, weights['hidden_layer']),\
biases['hidden_layer'])
layer_1 = tf.nn.relu(layer_1)
# Output layer with linear activation
logits = tf.add(tf.matmul(layer_1, weights['out']), biases['out'])
# Define loss and optimizer
cost = tf.reduce_mean(\
tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=y))
optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\
.minimize(cost)
# Initializing the variables
init = tf.global_variables_initializer()
# Launch the graph
with tf.Session() as sess:
sess.run(init)
# Training cycle
for epoch in range(training_epochs):
total_batch = int(info.splits["train"].num_examples/batch_size)
print(total_batch)
# Loop over all batches
for i in range(total_batch):
batch_x, batch_y = mnist_train.batch(batch_size)
# Run optimization op (backprop) and cost op (to get loss value)
sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})
When you load data using tfds.load you get an instance of tf.data.Dataset. You cannot feed this directly to a feed_dict but rather have to make an iterator and feed the values you obtain from the iterator at each step into the feedable input.You can do something roughly like this:
# one hot encode for 10 MNIST classes
def my_one_hot(feature, label):
return feature, tf.one_hot(label, depth=10)
# load your data from tfds
mnist_train, train_info = tfds.load(name="mnist", with_info=True, as_supervised=True, split=tfds.Split.TRAIN)
# convert your labels in one-hot
mnist_train = mnist_train.map(my_one_hot)
# you can batch your data here
mnist_train = mnist_train.batch(8)
Then you can launch your graph (and initialize your iterator)
# Launch the graph
with tf.Session() as sess:
sess.run(init)
# make an iterator
train_iterator = mnist_train.make_initializable_iterator()
next_element = train_iterator.get_next()
# Training cycle
for epoch in range(training_epochs):
sess.run(train_iterator.initializer)
batch_train_x, batch_train_y = sess.run(next_element)
total_batch = int(train_info.splits["train"].num_examples/batch_size)
print(total_batch)
# Loop over all batches
for i in range(total_batch):
# Run optimization op (backprop) and cost op (to get loss value)
sess.run(optimizer, feed_dict={x: batch_train_x, y: batch_train_y})
For more infor on tf.data.Dataset look at the guide. Hope it helps!
I'm just learning TensorFlow. The goal of this program is to detect Mines and rocks. But I have a problem when I feed the placeholders. I read many questions about this problem in this website but impossible to find a solution.
In the feed_dict the shape of train_y is (165,) and y (the placeholder) is (?, 2)... Here is the problem I think. But I don't know how to solve it. I tried to reshape train_y but it doesn't work.
I have this error:
ValueError: Cannot feed value of shape (165,) for Tensor
'Placeholder_11:0', which has shape '(?, 2)'
Here is the data and my program:
import pandas as pd
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
from sklearn.preprocessing import LabelEncoder
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split
#traitement des données
df = pd.read_csv('sonar.all-data.csv')
X = df[df.columns[:60]].values
y = df[df.columns[60]]
encoder = LabelEncoder()
encoder.fit(y)
y = encoder.transform(y)
#mix data
X,y = shuffle(X, y, random_state = 1)
#separate date for training
train_x, test_x, train_y, test_y = train_test_split(X, y, test_size = 0.2, random_state = 415)
# Parameters
learning_rate = 0.1
num_steps = 500
display_step = 100
# Network Parameters
n_hidden_1 = 60 # 1st layer number of neurons 60
n_hidden_2 = 60 # 2nd layer number of neurons 60
num_input = X.shape[1]
num_classes = 2
# tf Graph input
X = tf.placeholder("float", [None, num_input])
Y = tf.placeholder("float", [None, num_classes])
# Store layers weight & bias
weights = {
'h1': tf.Variable(tf.random_normal([num_input, n_hidden_1])),
'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])),
'out': tf.Variable(tf.random_normal([n_hidden_2, num_classes]))
}
biases = {
'b1': tf.Variable(tf.random_normal([n_hidden_1])),
'b2': tf.Variable(tf.random_normal([n_hidden_2])),
'out': tf.Variable(tf.random_normal([num_classes]))
}
# Create model
def neural_net(x):
# Hidden fully connected layer with 50 neurons
layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])
layer_1 = tf.nn.relu(layer_1)
# Hidden fully connected layer with 50 neurons
layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2'])
layer_2 = tf.nn.relu(layer_2)
# Output fully connected layer with a neuron for each class
out_layer = tf.matmul(layer_2, weights['out']) + biases['out']
out_layer = tf.nn.relu(out_layer)
return out_layer
# Construct model
logits = neural_net(X)
prediction = tf.nn.softmax(logits)
# Define loss and optimizer
loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
logits=logits, labels=Y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
train_op = optimizer.minimize(loss_op)
# Evaluate model
correct_pred = tf.equal(tf.argmax(prediction, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# Initialize the variables (i.e. assign their default value)
init = tf.global_variables_initializer()
# Start training
with tf.Session() as sess:
# Run the initializer
sess.run(init)
for step in range(1, num_steps+1):
sess.run(train_op, feed_dict={X: train_x, Y: train_y})
if step % display_step == 0 or step == 1:
# Calculate loss and accuracy
loss, acc = sess.run([loss_op, accuracy], feed_dict={X: train_x,
Y: train_y})
print("Step " + str(step) + ", Minibatch Loss= " + \
"{:.4f}".format(loss) + ", Training Accuracy= " + \
"{:.3f}".format(acc))
Thank you for your help!
I think you should apply one-hot encoding of your labels y: replace your LabelEncoder with sklearn.preprocessing.OneHotEncoder.
This code should work for your data:
y = df[df.columns[60]].apply(lambda x: 0 if x == 'R' else 1).values.reshape(-1, 1)
encoder = OneHotEncoder()
encoder.fit(y)
y = encoder.transform(y)
Below are the code i am using as a learning programming in Tensorflow.
from __future__ import print_function
from datetime import datetime
import time, os
import tensorflow as tf
# Import data
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
# Parameters
learning_rate = 0.001
training_epoch = 5
batch_size = 128
display_step = 10
model_path = "./output/model.ckpt"
logs_path = './logs'
directory = os.path.dirname(model_path)
if not os.path.exists(directory):
os.makedirs(directory)
directory = os.path.dirname(logs_path)
if not os.path.exists(directory):
os.makedirs(directory)
# Network Parameters
n_input = 784 # data input
n_classes = 10 # classes
dropout = 0.5 # Dropout, probability to keep units
l2_regularization_strength = 0.0005 #l2 regularization strength
# tf Graph input
x = tf.placeholder(tf.float32, [None, n_input], name='InputData')
y = tf.placeholder(tf.float32, [None, n_classes], name='LabelData')
keep_prob = tf.placeholder(tf.float32) #dropout (keep probability)
mode = tf.placeholder(tf.int32);
# Create some wrappers for simplicity
def conv2d(x, kernel_shape, strides=1, mode=0):
# Conv2D wrapper, with batch normalization and relu activation
weights = tf.get_variable('weights', kernel_shape, initializer=tf.contrib.layers.xavier_initializer())
x = tf.nn.conv2d(x, weights, strides=[1, strides, strides, 1], padding='SAME')
pop_mean = tf.get_variable('bn_pop_mean', [x.get_shape()[-1]], initializer=tf.constant_initializer(0), trainable=False)
pop_var = tf.get_variable('bn_pop_var', [x.get_shape()[-1]], initializer=tf.constant_initializer(1), trainable=False)
scale = tf.get_variable('bn_scale', [x.get_shape()[-1]], initializer=tf.constant_initializer(1))
beta = tf.get_variable('bn_beta', [x.get_shape()[-1]], initializer=tf.constant_initializer(0))
epsilon = 1e-3
decay = 0.999
if mode == 0:
batch_mean, batch_var = tf.nn.moments(x,[0, 1, 2])
train_mean = tf.assign(pop_mean, pop_mean * decay + batch_mean * (1 - decay))
train_var = tf.assign(pop_var, pop_var * decay + batch_var * (1 - decay))
with tf.control_dependencies([train_mean, train_var]):
bn = tf.nn.batch_normalization(x, batch_mean, batch_var, beta, scale, epsilon, name='bn')
else:
bn = tf.nn.batch_normalization(x, pop_mean, pop_var, beta, scale, epsilon, name='bn')
return tf.nn.relu(bn, name = 'relu')
def maxpool2d(x, k=2):
# MaxPool2D wrapper
return tf.nn.max_pool(x, ksize=[1, k, k, 1], strides=[1, k, k, 1], padding='SAME', name='maxpool')
# Create model
def conv_net(x, dropout, mode):
# Reshape input picture
x = tf.reshape(x, shape=[-1, 28, 28, 1])
with tf.variable_scope("conv1"):
# Convolution Layer
conv1 = conv2d(x, [5, 5, 1, 32], mode=mode)
# Max Pooling (down-sampling)
conv1 = maxpool2d(conv1, k=2)
with tf.variable_scope("conv2"):
# Convolution Layer
conv2 = conv2d(conv1, [5, 5, 32, 64], mode=mode)
# Max Pooling (down-sampling)
conv2 = maxpool2d(conv2, k=2)
with tf.variable_scope("fc1"):
# Fully connected layer
# Reshape conv2 output to fit fully connected layer input
weights = tf.get_variable("weights", [7*7*64, 1024], initializer=tf.contrib.layers.xavier_initializer())
biases = tf.get_variable("biases", [1024], initializer=tf.constant_initializer(0.0))
fc1 = tf.reshape(conv2, [-1, weights.get_shape().as_list()[0]])
fc1 = tf.add(tf.matmul(fc1, weights), biases)
fc1 = tf.nn.relu(fc1, name = 'relu')
# Apply Dropout
fc1 = tf.nn.dropout(fc1, dropout, name='dropout')
with tf.variable_scope("output"):
# Output, class prediction
weights = tf.get_variable("weights", [1024, n_classes], initializer=tf.contrib.layers.xavier_initializer())
biases = tf.get_variable("biases", [n_classes], initializer=tf.constant_initializer(0.0))
out = tf.add(tf.matmul(fc1, weights), biases)
return out
with tf.name_scope('Model'):
# Construct model
pred = conv_net(x, keep_prob, mode)
with tf.name_scope('Loss'):
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y))
vars = tf.trainable_variables()
l2_regularization = tf.add_n([tf.nn.l2_loss(v) for v in vars if any(x in v.name for x in ['weights', 'biases'])])
for v in vars:
if any(x in v.name for x in ['weights', 'biases']):
print(v.name + '-included!')
else:
print(v.name)
cost += l2_regularization_strength*l2_regularization
with tf.name_scope('Optimizer'):
# Define optimizer
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
# Op to calculate every variable gradient
grads = tf.gradients(cost, tf.trainable_variables())
grads = list(zip(grads, tf.trainable_variables()))
# Op to update all variables according to their gradient
apply_grads = optimizer.apply_gradients(grads_and_vars=grads)
with tf.name_scope('Accuracy'):
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# Initializing the variables
init = tf.initialize_all_variables()
# Create a summary to monitor cost tensor
tf.scalar_summary('cost', cost)
# Create a summary to monitor l2_regularization tensor
tf.scalar_summary('l2_regularization', l2_regularization)
# Create a summary to monitor accuracy tensor
tf.scalar_summary('accuracy', accuracy)
# Create summaries to visualize weights
for var in tf.trainable_variables():
tf.histogram_summary(var.name, var)
for var in tf.all_variables():
if 'bn_pop' in var.name:
tf.histogram_summary(var.name, var)
# Summarize all gradients
for grad, var in grads:
tf.histogram_summary(var.name + '/gradient', grad)
# Merge all summaries into a single op
merged_summary_op = tf.merge_all_summaries()
# 'Saver' op to save and restore all the variables
saver = tf.train.Saver()
# Launch the graph
with tf.Session() as sess:
sess.run(init)
step = 1
# op to write logs to Tensorboard
summary_writer = tf.train.SummaryWriter(logs_path, graph=tf.get_default_graph())
# Keep training until reach max epoch
while step * batch_size < training_epoch * mnist.train.num_examples:
start_time = time.time()
# Get barch
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Run optimization op (backprop)
sess.run(apply_grads, feed_dict={x: batch_x, y: batch_y, keep_prob: dropout, mode: 0})
duration = time.time() - start_time
if step % display_step == 0:
# Calculate batch loss and accuracy
loss, acc, summary = sess.run([cost, accuracy, merged_summary_op], feed_dict={x: batch_x,
y: batch_y,
keep_prob: 1.,
mode: 1})
# Write logs at every iteration
summary_writer.add_summary(summary, step)
# Calculate number sample per sec
samples_per_sec = batch_size / duration
format_str = ('%s: Iter %d, Epoch %d, (%.1f examples/sec; %.3f sec/batch), Minibatch Loss = %.5f , Training Accuracy=%.5f')
print (format_str % (datetime.now(), step*batch_size, int(step*batch_size/mnist.train.num_examples) + 1, samples_per_sec, float(duration), loss, acc))
step += 1
print("Optimization Finished!")
# Calculate accuracy for 256 mnist test images
print("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: mnist.test.images[:5000],
y: mnist.test.labels[:5000],
keep_prob: 1.,
mode: 2}))
# Save model weights to disk
save_path = saver.save(sess, model_path)
print("Model saved in file: %s" % save_path)
When i open the tensorboard and look at the histogram and distribution sesstion, the 'bn_pop_mean' and 'bn_pop_var' in 'conv1' and 'conv2' are not updateing (they are constant at the initialised value).
Although after the training i achieved around 97% accuracy, i don't know if it the batch normalization is in effect.
In your conv_net function, you didn't set the "reuse" parameter for the tf.variable_scope(). The default setting for "reuse" is "None". Every time conv2d function is called, "bn_pop_mean" and "bn_pop_var" are re-initalized.
if mode == 0:
batch_mean, batch_var = tf.nn.moments(x,[0, 1, 2])
train_mean = tf.assign(pop_mean, pop_mean * decay + batch_mean * (1 - decay))
train_var = tf.assign(pop_var, pop_var * decay + batch_var * (1 - decay))
with tf.control_dependencies([train_mean, train_var]):
bn = tf.nn.batch_normalization(x, batch_mean, batch_var, beta, scale, epsilon, name='bn')
else:
bn = tf.nn.batch_normalization(x, pop_mean, pop_var, beta, scale, epsilon, name='bn')
It seems that the if prediction here always evaluate to be False. I guess what you want to do is using mode via feed_dict to control your batch normalization. So you should use tf.cond in TensorFlow instead of if in Python.
I am implementing a network using TensorFlow.
The network take as input a binary feature vector, and it should predict a float value as output.
I am expecting a (1,1) tensor object as output for my function multilayer_perceptron(), instead, when running pred, it returns a vector of the same length of my input data (X,1).
Since i am new to this framework I expect the error to be very trivial.
What am I doing wrong?
import tensorflow as tf
print "**** Defining parameters..."
# Parameters
learning_rate = 0.001
training_epochs = 15
batch_size = 1
display_step = 1
print "**** Defining Network..."
# Network Parameters
n_hidden_1 = 10 # 1st layer num features
n_hidden_2 = 10 # 2nd layer num features
n_input = Xa.shape[1] # data input(feature vector length)
n_classes = 1 # total classes (IC50 value)
# tf Graph input
x = tf.placeholder("int32", [batch_size, None])
y = tf.placeholder("float", [None, n_classes])
# Create model
def multilayer_perceptron(_X, _weights, _biases):
lookup_h1 = tf.nn.embedding_lookup(_weights['h1'], _X)
layer_1 = tf.nn.relu(tf.add(tf.reduce_sum(lookup_h1, 0), _biases['b1'])) #Hidden layer with RELU activation
layer_2 = tf.nn.relu(tf.add(tf.matmul(layer_1, _weights['h2']), _biases['b2'])) #Hidden layer with RELU activation
pred = tf.matmul(layer_2, _weights['out']) + _biases['out']
return pred
# Store layers weight & bias
weights = {
'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1])),
'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])),
'out': tf.Variable(tf.random_normal([n_hidden_2, n_classes]))
}
biases = {
'b1': tf.Variable(tf.random_normal([n_hidden_1])),
'b2': tf.Variable(tf.random_normal([n_hidden_2])),
'out': tf.Variable(tf.random_normal([n_classes]))
}
# Construct model
pred = multilayer_perceptron(x, weights, biases)
# Define loss and optimizer
cost = tf.reduce_mean(tf.square(tf.sub(pred, y))) # MSE
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost) #Gradient descent
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# Initializing the variables
init = tf.initialize_all_variables()
print "**** Launching the graph..."
# Launch the graph
with tf.Session() as sess:
sess.run(init)
print "**** Training..."
# Training cycle
for epoch in range(training_epochs):
avg_cost = 0.
total_batch = int(Xa.tocsc().shape[0]/batch_size)
# Loop over all batches
for i in range(total_batch):
# Extract sample
batch_xs = Xa.tocsc()[i,:].tocoo()
batch_ys = np.reshape(Ya.tocsc()[i,0], (batch_size,1))
#****************************************************************************
# Extract sparse indeces from input matrix (They will be used as actual input)
ids = batch_xs.nonzero()[1]
# Fit training using batch data
sess.run(optimizer, feed_dict={x: ids, y: batch_ys})
# Compute average loss
avg_cost += sess.run(cost, feed_dict={x: ids, y: batch_ys})/total_batch
# Display logs per epoch step
if epoch % display_step == 0:
print "Epoch:", '%04d' % (epoch+1), "cost=", "{:.9f}".format(avg_cost)
print "Optimization Finished!"
Your pred should be of shape [n_input, n_class] because you define weights['out'] and biases['out'] in that way. The only way you get a (1,1) tensor from pred is your n_class = 1...
I'm trying to train a RNN-Classifier with a training set that has variable length, I've read around and the suggested solution is to do bucketing on the training data and then feed batches out of each bucket and exit the RNN on certain sequence_length, however the results from this process vs just trimming the input sequences to length 10 is extremely worse and my loss function and accuracy move up and down erratically
Some more information:
I have variable batch size because some times there's not enough samples of a certain sequence length in a bucket
A sequence is a series of words converted into word embeddings
If I have a bucket of sequences of length 4, the batch would be of size max_seq_length padding everything on the right with vectors full of 0's
batches have the following shape (batch_size,number_steps,embedding_dimensions)
batch(128,48,100)
sequence_length = 4
classes(128,188)
batch(128,48,100)
sequence_length = 8
classes(128,188)
...
batch(30,48,100)
sequence_length = 40
classes(30,188)
I feed the sequence_length as the early_stop variable to my RNN
What is the best way to mix variable sequence length with variable bath size?
My Graph:
# Network Parameters
n_input = embeddings_dim # word embeddings dimensions : 100
n_steps = max_seq_len # timesteps = maximum sequence length in my training = 47
n_classes = total_labels # total classes = 188
graph = tf.Graph()
with graph.as_default():
# tf Graph input
x = tf.placeholder("float", [None, n_steps, n_input])
# Tensorflow LSTM cell requires 2x n_hidden length (state & cell)
istate = tf.placeholder("float", [None, 2*n_hidden])
y = tf.placeholder("float", [None, n_classes])
#at what step we should read out the value of the RNN
early_stop = tf.placeholder(tf.int32)
tf.scalar_summary('early_stop', early_stop)
# Define weights
weights = {
'hidden': tf.Variable(tf.random_normal([n_input, n_hidden])), # Hidden layer weights
'out': tf.Variable(tf.random_normal([n_hidden, n_classes]))
}
biases = {
'hidden': tf.Variable(tf.random_normal([n_hidden])),
'out': tf.Variable(tf.random_normal([n_classes]))
}
def RNN(_X, _istate, _weights, _biases):
# input shape: (batch_size, n_steps, n_input)
_X = tf.transpose(_X, [1, 0, 2]) # permute n_steps and batch_size
# Reshape to prepare input to hidden activation
_X = tf.reshape(_X, [-1, n_input]) # (n_steps*batch_size, n_input)
# Linear activation
_X = tf.matmul(_X, _weights['hidden']) + _biases['hidden']
# Define a lstm cell with tensorflow
lstm_cell = rnn_cell.LSTMCell(n_hidden, forget_bias=0.25)
# Split data because rnn cell needs a list of inputs for the RNN inner loop
_X = tf.split(0, n_steps, _X) # n_steps * (batch_size, n_hidden)
# Get lstm cell output
outputs, states = rnn.rnn(lstm_cell, _X,
initial_state=_istate,
sequence_length=early_stop)
# Linear activation
# Get inner loop last output
return tf.matmul(outputs[-1], _weights['out']) + _biases['out']
pred = RNN(x, istate, weights, biases)
# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y)) # Softmax loss
global_step = tf.Variable(0, name='global_step', trainable=False)
learning_rate_2 = tf.train.exponential_decay(learning_rate, global_step, 300, 0.96, staircase=True)
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate_2).minimize(cost) # Adam Optimizer
# Evaluate model
correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
My Training loop:
with tf.Session(graph=graph) as sess:
init = tf.initialize_all_variables()
sess.run(init)
step = 1
while step * batch_size < training_iters:
#batch_xs = batch_xs.reshape((batch_size, n_steps, n_input))
batch_xs, batch_ys, batch_sl = train_batches.next()
ins_batch_size = len(batch_xs)
# Fit training using batch data
summ ,acc = sess.run([merged,optimizer], feed_dict={x: batch_xs, y: batch_ys,
istate: np.zeros((ins_batch_size, 2*n_hidden)),
early_stop:batch_sl})
train_writer.add_summary(summ, step)
if step % display_step == 0:
# Calculate batch accuracy
acc = sess.run(accuracy, feed_dict={x: batch_xs, y: batch_ys,
istate: np.zeros((ins_batch_size, 2*n_hidden)),
early_stop:batch_sl})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_xs, y: batch_ys,
istate: np.zeros((ins_batch_size, 2*n_hidden)),
early_stop:batch_sl})
print( "Iter " + str(step*batch_size) + ", Minibatch Loss= " + "{:.6f}".format(loss) + \
", Training Accuracy= " + "{:.5f}".format(acc))
step += 1
print( "Optimization Finished!")
test_data,test_label, test_sl = test_batches.next()
test_len = len(test_data)
print( "Testing Accuracy:", sess.run(accuracy, feed_dict={x: test_data, y: test_label,
istate: np.zeros((test_len, 2*n_hidden)),
early_stop:test_sl}))