I am still trying to grasp how to restore a saved tensorflow graph from disk and feed through dictionaries to the model. I have looked at multiple sources but cannot troubleshoot this. The generic MLP code below (first snippet) saves files to disk, however after restoring (second snippet), my accuracy returns a value of None. Any ideas what the reason for this may be?
# Import MINST data
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
import tensorflow as tf
# Parameters
learning_rate = 0.001
training_epochs = 15
batch_size = 100
display_step = 1
# Network Parameters
n_hidden_1 = 256 # 1st layer number of features
n_hidden_2 = 256 # 2nd layer number of features
n_input = 784 # MNIST data input (img shape: 28*28)
n_classes = 10 # MNIST total classes (0-9 digits)
with tf.name_scope('placeholders'):
# tf Graph input
x = tf.placeholder("float", [None, n_input],name='x')
y = tf.placeholder("float", [None, n_classes],name='y')
with tf.name_scope('Layer-1'):
NN_weights_1=tf.Variable(tf.random_normal([n_input, n_hidden_1],seed=1),name='NN_weights_1')
NN_biases_1=tf.Variable(tf.constant(0.0,shape=[n_hidden_1],name='Const'),name='NN_biases_1')
func=tf.add(tf.matmul(x, NN_weights_1,name='matmul'), NN_biases_1,name='Addition')
func_2=tf.nn.relu(func)
with tf.name_scope('Layer-2'):
NN_weights_2=tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2],seed=2),name='NN_weights_2')
NN_biases_2=tf.Variable(tf.constant(0.0,shape=[n_hidden_2],name='Const'),name='NN_biases_2')
func_3=tf.add(tf.matmul(func_2, NN_weights_2,name='matmul'), NN_biases_2,name='Addition')
func_4=tf.nn.relu(func_3)
with tf.name_scope('Output'):
NN_weights_3=tf.Variable(tf.random_normal([n_hidden_2, n_classes],seed=3),name='NN_weights_3')
NN_biases_3=tf.Variable(tf.constant(0.0,shape=[n_classes],name='Const'),name='NN_biases_3')
func_3=tf.add(tf.matmul(func_4, NN_weights_3,name='matmul'), NN_biases_3,name='Addition')
func_4=tf.nn.sigmoid(func_3)
# Define loss and optimizer
with tf.name_scope('Operations_'):
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=func_4, labels=y),name='cost')
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
# Test model
correct_prediction = tf.equal(tf.argmax(func_4, 1), tf.argmax(y, 1),name='correct_prediction')
# Calculate accuracy
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"),name='accuracy')
# Initializing the variables
init = tf.global_variables_initializer()
# Launch the graph
with tf.Session() as sess:
sess.run(init)
saver = tf.train.Saver()
# Training cycle
for epoch in range(training_epochs):
avg_cost = 0.
total_batch = int(mnist.train.num_examples/batch_size)
# Loop over all batches
for i in range(total_batch):
batch_x, batch_y = mnist.train.next_batch(batch_size)
# Run optimization op (backprop) and cost op (to get loss value)
_, c = sess.run([optimizer, cost], feed_dict={x: batch_x,
y: batch_y})
# Compute average loss
avg_cost += c / total_batch
# Display logs per epoch step
if epoch % display_step == 0:
print (("Epoch:", '%04d' % (epoch+1), "cost="), \
"{:.9f}".format(avg_cost))
print ("Optimization Finished!")
print ("Accuracy:", accuracy.eval({x: mnist.test.images, y: mnist.test.labels}))
saver.save(sess, 'my_test_model',global_step=1000)
Restoring Model and passing dictionary for accuracy:
import tensorflow as tf
sess=tf.Session()
#First let's load meta graph and restore weights
saver = tf.train.import_meta_graph('my_test_model-1000.meta')
saver.restore(sess,"my_test_model-1000")
graph = tf.get_default_graph()
accuracy=graph.get_operation_by_name("Operations_/accuracy")
# Access saved Variables directly
print(sess.run('Layer-1/NN_weights_1:0'))
# This will print 2, which is the value of bias that we saved
print ("Accuracy:", sess.run([accuracy],feed_dict={'placeholders/x:0': mnist.test.images, 'placeholders/y:0': mnist.test.labels}))
Change it to:
accuracy=graph.get_operation_by_name("Operations_/accuracy").outputs[0]
Tensorflow discards the outputs of Operation objects executed by means of Session.run. See here for detailed explanation: TensorFlow: eval restored graph
Related
I have written the following Tensorflow code that performs logistic regression on a custom dataset.
def logi_regression(data, labels, test_data, test_labels, learning_rate,
batch_size, training_epochs, display_step):
x = tf.placeholder(tf.float32, [None, data.shape[1]])
y = tf.placeholder(tf.float32, [None, labels.shape[1]])
# Weights
W = tf.Variable(tf.zeros([data.shape[1], 1]))
b = tf.Variable(tf.zeros([1, 1]))
# Logistic Model
pred = tf.nn.sigmoid(tf.matmul(x, W) + b)
# Error function
loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=pred,
labels=y))
# Gradient Descent
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)
# Initialise global variables
init = tf.global_variables_initializer()
init_l = tf.local_variables_initializer()
# Training
with tf.Session() as sess:
# Run the initializer
sess.run(init)
sess.run(init_l)
# Training cycle
for epoch in range(training_epochs):
avg_cost = 0.
total_batch = int(data.shape[0]/batch_size)
# Loop over all batches
for i in range(total_batch):
# The next_data_batch is a custom made function
batch_xs, batch_ys = next_data_batch(batch_size,
data, labels)
# Run optimization op (backprop) and cost op (to get loss value)
_, c = sess.run([optimizer, loss], feed_dict={x: batch_xs,
y: batch_ys})
# Compute average loss
avg_cost += c / total_batch
# Display logs per epoch step
if (epoch+1) % display_step == 0:
print("Epoch:", '%04d' % (epoch+1), "cost=", "{:.9f}".format(avg_cost))
print("Optimization Finished!")
# Test model
prediction = tf.round(tf.sigmoid(pred))
correct = tf.cast(tf.equal(prediction, y), dtype=tf.float32)
_, precision = tf.metrics.precision(y, prediction)
# Calculate accuracy
accuracy = tf.reduce_mean(correct)
avg_prec = tf.reduce_mean(precision)
print("Accuracy:", accuracy.eval({x: test_data, y: test_labels}))
print("Average Precision Score:", avg_prec.eval({x: test_data, y: test_labels}))
But, even though I get the correct output from the training (Epoch: xxxx cost= 0.xxxx) and from the test set (Accuracy:0.xxx). When the program tries to calculate the precision it returns an error:
FailedPreconditionError (see above for traceback): Attempting to use
uninitialized value precision/true_positives/count [[node
precision/true_positives/AssignAdd (defined at
:54) ]]
Therefore, the problem is in the final lines that I have added the (_, precision = tf.metrics.precision(y, prediction)). I have tried various suggestions from Stackoverflow posts but nothing has worked. It must be a silly coding mistake but due to my inexperience with Tensorflow I cannot figure out what it is.
The lines creating nodes in the tensorflow-graph should be before the 'tf.global_variables_initializer()' statement to be part of the default graph. Move the following lines to above the initializer and it will work:
# Test model
prediction = tf.round(tf.sigmoid(pred))
correct = tf.cast(tf.equal(prediction, y), dtype=tf.float32)
_, precision = tf.metrics.precision(y, prediction)
# Calculate accuracy
accuracy = tf.reduce_mean(correct)
avg_prec = tf.reduce_mean(precision)
I am trying to execute the following code which is using MNIST dataset in Tensorflow, with images of shape 28 * 28 = 784 and 10 classes (0-9 digits) as output, I am getting an error that is showed as follows :
InvalidArgumentError: You must feed a value for placeholder tensor 'Placeholder_33' with dtype float and shape [?,10]
# Import MNIST data
#import input_data
#mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
import tensorflow as tf
# Set parameters
learning_rate = 0.01
training_iteration = 30
batch_size = 100
display_step = 2
# TF graph input
x = tf.placeholder("float", [None, 784]) # mnist data image of shape 28*28=784
y = tf.placeholder("float", [None, 10]) # 0-9 digits recognition => 10 classes
# Create a model
# Set model weights
W = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))
with tf.name_scope("Wx_b") as scope:
# Construct a linear model
model = tf.nn.softmax(tf.matmul(x, W) + b) # Softmax
# Add summary ops to collect data
w_h = tf.summary.histogram("weights", W)
b_h = tf.summary.histogram("biases", b)
# More name scopes will clean up graph representation
with tf.name_scope("cost_function") as scope:
# Minimize error using cross entropy
# Cross entropy
cost_function = -tf.reduce_sum(y*tf.log(model))
# Create a summary to monitor the cost function
tf.summary.scalar("cost_function", cost_function)
with tf.name_scope("train") as scope:
# Gradient descent
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost_function)
# Initializing the variables
init = tf.initialize_all_variables()
# Merge all summaries into a single operator
merged_summary_op = tf.summary.merge_all()
# Launch the graph
with tf.Session() as sess:
sess.run(init)
summary_writer = tf.summary.FileWriter('/home/raed/Tensorflow/tensorflow_demo', graph_def=sess.graph_def)
# Training cycle
for iteration in range(training_iteration):
avg_cost = 0.
total_batch = int(mnist.train.num_examples/batch_size)
# Loop over all batches
for i in range(total_batch):
batch_xs, batch_ys = mnist.train.next_batch(batch_size)
# Fit training using batch data
sess.run(optimizer, feed_dict={x: batch_xs, y: batch_ys})
# Compute the average loss
avg_cost += sess.run(cost_function, feed_dict={x: batch_xs, y: batch_ys})/total_batch
# Write logs for each iteration
summary_str = sess.run(merged_summary_op, feed_dict={x: batch_xs, y: batch_ys})
summary_writer.add_summary(summary_str, iteration*total_batch + i)
# Display logs per iteration step
if iteration % display_step == 0:
print ("Iteration:" "%04d" % (iteration + 1), "cost=", "{:.9f}".format(avg_cost))
print ("Tuning completed!")
# Test the model
predictions = tf.equal(tf.argmax(model, 1), tf.argmax(y, 1))
# Calculate accuracy
accuracy = tf.reduce_mean(tf.cast(predictions, "float"))
print ("Accuracy:", accuracy.eval({x: mnist.test.images, y: mnist.test.labels}))
I am working through some code to understand how to save and restore checkpoints in tensorflow. To do so, I implemented a simple neural netowork that works with MNIST digits and saved the .ckpt file like so:
from tensorflow.examples.tutorials.mnist import input_data
import numpy as np
learning_rate = 0.001
n_input = 784 # MNIST data input (img shape = 28*28)
n_classes = 10 # MNIST total classes 0-9
#import MNIST data
mnist = input_data.read_data_sets('.', one_hot = True)
#Features and Labels
features = tf.placeholder(tf.float32, [None, n_input])
labels = tf.placeholder(tf.float32, [None, n_classes])
#Weights and biases
weights = tf.Variable(tf.random_normal([n_input, n_classes]))
bias = tf.Variable(tf.random_normal([n_classes]))
#logits = xW + b
logits = tf.add(tf.matmul(features, weights), bias)
#Define loss and optimizer
cost = tf.reduce_mean(\
tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels))
optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\
.minimize(cost)
# Calculate accuracy
correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(labels, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
import math
save_file = './train_model.ckpt'
batch_size = 128
n_epochs = 100
saver = tf.train.Saver()
# Launch the graph
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
# Training cycle
for epoch in range(n_epochs):
total_batch = math.ceil(mnist.train.num_examples / batch_size)
# Loop over all batches
for i in range(total_batch):
batch_features, batch_labels = mnist.train.next_batch(batch_size)
sess.run(
optimizer,
feed_dict={features: batch_features, labels: batch_labels})
# Print status for every 10 epochs
if epoch % 10 == 0:
valid_accuracy = sess.run(
accuracy,
feed_dict={
features: mnist.validation.images,
labels: mnist.validation.labels})
print('Epoch {:<3} - Validation Accuracy: {}'.format(
epoch,
valid_accuracy))
# Save the model
saver.save(sess, save_file)
print('Trained Model Saved.')
This part works well, and I get the .ckpt file saved in the correct directory. The problem comes in when I try to restore the model in an attempt to work on it again. I use the following code to restore the model:
saver = tf.train.Saver()
with tf.Session() as sess:
saver.restore(sess, 'train_model.ckpt.meta')
print('model restored')
and end up with the error: ValueError: No variables to save
Not too sure, what the mistake here is. Any help is appreciated. Thanks in advance
A Graph is different to the Session. A graph is the set of operations joining tensors, each of which is a symbolic representation of a set of values. A Session assigns specific values to the Variable tensors, and allows you to run operations in that graph.
The chkpt file saves variable values - i.e. those saved in the weights and biases - but not the graph itself.
The solution is simple: re-run the graph construction (everything before the Session, then start your session and load values from the chkpt file.
Alternatively, you can check out this guide for exporting and importing MetaGraphs.
You should tell the Saver which Variables to restore, default Saver will get all the Variables from the default graph.
As in your case, you should add the constructing graph code before saver = tf.train.Saver()
I was making a computer to predict a hand-written number from MNist data set using softmax function. and something weird happened. the cost was decreasing over time and eventually becomes something around 0.0038....(I used softmax_crossentropy_with_logits() for the cost function) However, the accuracy was pretty as low as 33%. So I thought "well.. I don't know what happened there but if I do softmax and crossentropy separately maybe it will produce different result !" and boom ! accuracy went up to 89 %. I have no idea why doing softmax and crossentropy separately makes such different result. I even looked up here :difference between tensorflow tf.nn.softmax and tf.nn.softmax_cross_entropy_with_logits
so this is the code that I used softmax_cross_entropy_with_logits() for the cost function (accuracy: 33%)
import tensorflow as tf
import numpy as np
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
X = tf.placeholder(shape=[None,784],dtype=tf.float32)
Y = tf.placeholder(shape=[None,10],dtype=tf.float32)
W1= tf.Variable(tf.random_normal([784,20]))
b1= tf.Variable(tf.random_normal([20]))
layer1 = tf.nn.softmax(tf.matmul(X,W1)+b1)
W2 = tf.Variable(tf.random_normal([20,10]))
b2 = tf.Variable(tf.random_normal([10]))
logits = tf.matmul(layer1,W2)+b2
hypothesis = tf.nn.softmax(logits) # just so I can figure our the accuracy
cost_i= tf.nn.softmax_cross_entropy_with_logits(logits=logits,labels=Y)
cost = tf.reduce_mean(cost_i)
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01).minimize(cost)
batch_size = 100
train_epoch = 25
display_step = 1
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
for epoch in range(train_epoch):
av_cost = 0
total_batch = int(mnist.train.num_examples / batch_size)
for batch in range(total_batch):
batch_xs, batch_ys = mnist.train.next_batch(batch_size)
sess.run(optimizer,feed_dict={X:batch_xs,Y:batch_ys})
av_cost += sess.run(cost,feed_dict={X:batch_xs,Y:batch_ys})/total_batch
if epoch % display_step == 0: # Softmax
print ("Epoch:", '%04d' % (epoch + 1), "cost=", "{:.9f}".format(av_cost))
print ("Optimization Finished!")
correct_prediction = tf.equal(tf.argmax(hypothesis,1),tf.argmax(Y,1))
accuray = tf.reduce_mean(tf.cast(correct_prediction,'float32'))
print("Accuracy:",sess.run(accuray,feed_dict={X:mnist.test.images,Y:mnist.test.labels}))
and this is the one that I did softmax and cross_entropy separately(accuracy: 89%)
import tensorflow as tf #89 % accuracy one
import numpy as np
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
X = tf.placeholder(shape=[None,784],dtype=tf.float32)
Y = tf.placeholder(shape=[None,10],dtype=tf.float32)
W1= tf.Variable(tf.random_normal([784,20]))
b1= tf.Variable(tf.random_normal([20]))
layer1 = tf.nn.softmax(tf.matmul(X,W1)+b1)
W2 = tf.Variable(tf.random_normal([20,10]))
b2 = tf.Variable(tf.random_normal([10]))
#logits = tf.matmul(layer1,W2)+b2
#cost_i= tf.nn.softmax_cross_entropy_with_logits(logits=logits,labels=Y)
logits = tf.matmul(layer1,W2)+b2
hypothesis = tf.nn.softmax(logits)
cost = tf.reduce_mean(tf.reduce_sum(-Y*tf.log(hypothesis)))
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01).minimize(cost)
batch_size = 100
train_epoch = 25
display_step = 1
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
for epoch in range(train_epoch):
av_cost = 0
total_batch = int(mnist.train.num_examples / batch_size)
for batch in range(total_batch):
batch_xs, batch_ys = mnist.train.next_batch(batch_size)
sess.run(optimizer,feed_dict={X:batch_xs,Y:batch_ys})
av_cost += sess.run(cost,feed_dict={X:batch_xs,Y:batch_ys})/total_batch
if epoch % display_step == 0: # Softmax
print ("Epoch:", '%04d' % (epoch + 1), "cost=", "{:.9f}".format(av_cost))
print ("Optimization Finished!")
correct_prediction = tf.equal(tf.argmax(hypothesis,1),tf.argmax(Y,1))
accuray = tf.reduce_mean(tf.cast(correct_prediction,'float32'))
print("Accuracy:",sess.run(accuray,feed_dict={X:mnist.test.images,Y:mnist.test.labels}))
If you use tf.reduce_sum() in the upper example, as you did in the lower one, you should be able to achieve similar results with both methods: cost = tf.reduce_mean(tf.reduce_sum( tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=Y))).
I increased the number of training epochs to 50 and achieved accuracies of 93.06% (tf.nn.softmax_cross_entropy_with_logits()) and 93.24% (softmax and cross entropy separately), so the results are quite similar.
From Tensorflow API here the second way, cost = tf.reduce_mean(tf.reduce_sum(-Y*tf.log(hypothesis))) is numerically unstable, and because of that you can't get same results,
Whatever, you can find on my GitHub the implementation of numerically stable cross entropy loss function which has the same result as tf.nn.softmax_cross_entropy_with_logits() function.
You can see that tf.nn.softmax_cross_entropy_with_logits() doesn't calculate the large numbers softmax normalization, on only approximate them, more details are in README section.
I implemented a relatively straightforward logistic regression function. I save all the necessary variables such as weights, bias, x, y, etc. and then I run the training algorithm...
# launch the graph
with tf.Session() as sess:
sess.run(init)
# training cycle
for epoch in range(FLAGS.training_epochs):
avg_cost = 0
total_batch = int(mnist.train.num_examples/FLAGS.batch_size)
# loop over all batches
for i in range(total_batch):
batch_xs, batch_ys = mnist.train.next_batch(FLAGS.batch_size)
_, c = sess.run([optimizer, cost], feed_dict={x: batch_xs, y: batch_ys})
# compute average loss
avg_cost += c / total_batch
# display logs per epoch step
if (epoch + 1) % FLAGS.display_step == 0:
print("Epoch:", '%04d' % (epoch + 1), "cost=", "{:.9f}".format(avg_cost))
save_path = saver.save(sess, "/tmp/model.ckpt")
The model is saved and the prediction and accuracy of the trained model is displayed...
# list of booleans to determine the correct predictions
correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
print(correct_prediction.eval({x:mnist.test.images, y:mnist.test.labels}))
# calculate total accuracy
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
print("Accuracy:", accuracy.eval({x: mnist.test.images, y: mnist.test.labels}))
This is all fine and dandy. However, now I want to be able to predict any given image using the trained model. For example, I want to feed it picture of say 7 and see what it predicts it to be.
I have another module that restores the model. First we load the variables...
mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
# tf Graph Input
x = tf.placeholder(tf.float32, [None, 784]) # mnist data image of shape 28*28=784
y = tf.placeholder(tf.float32, [None, 10]) # 0-9 digits recognition => 10 classes
# set model weights
W = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))
# construct model
pred = tf.nn.softmax(tf.matmul(x, W) + b) # Softmax
# minimize error using cross entropy
cost = tf.reduce_mean(-tf.reduce_sum(y * tf.log(pred), reduction_indices=1))
# Gradient Descent
optimizer = tf.train.GradientDescentOptimizer(FLAGS.learning_rate).minimize(cost)
# initializing the variables
init = tf.global_variables_initializer()
saver = tf.train.Saver()
with tf.Session() as sess:
save.restore(sess, "/tmp/model.ckpt")
This is good. Now I want to compare one image to the model and get a prediction. In this example, I take the first image from the test dataset mnist.test.images[0] and I attempt to compare it to the model.
classification = sess.run(tf.argmax(pred, 1), feed_dict={x: mnist.test.images[0]})
print(classification)
I know this will not work. I get the error...
ValueError: Cannot feed value of shape (784,) for Tensor 'Placeholder:0', which has shape '(?, 784)'
I am at a loss for ideas. This question is rather long, if a straightforward answer is not possible, some guidance as to the steps I may take to do this is appreciated.
Your input placeholder must be of size (?, 784), the question mark meaning variable size which is probably the batch size. You are feeding an input of size (784,) which does not work as the error message states.
In your case, during prediction time, the batch size is just 1, so the following should work:
import numpy as np
...
x_in = np.expand_dims(mnist.test.images[0], axis=0)
classification = sess.run(tf.argmax(pred, 1), feed_dict={x:x_in})
Assuming that the input image is available as a numpy array. If it is already a tensor, the corresponding function is tf.expand_dims(..).