We Keep Coding

Can someone help me with writing a piece of code in Python regarding neural networks and the MNIST dataset?

For a schoolproject I have analysed the code beneath, but I want to put a feature to it: I want to give the neural network, when its done with training, an image of a handwritten digit from MNIST (lets say an 8) so that it can try and define the number 8. Because Im totally new with coding and machine learning, although I really like it and want to learn more, I could not figure it out myself how such a code should look like. Can someone help me?
The code is written in Python:
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
learning_rate = 0.0001
batch_size = 100
update_step = 10
layer_1_nodes = 500
layer_2_nodes = 500
layer_3_nodes = 500
output_nodes = 10
network_input = tf.placeholder(tf.float32, [None, 784])
target_output = tf.placeholder(tf.float32, [None, output_nodes])
layer_1 = tf.Variable(tf.random_normal([784, layer_1_nodes]))
layer_1_bias = tf.Variable(tf.random_normal([layer_1_nodes]))
layer_2 = tf.Variable(tf.random_normal([layer_1_nodes, layer_2_nodes]))
layer_2_bias = tf.Variable(tf.random_normal([layer_2_nodes]))
layer_3 = tf.Variable(tf.random_normal([layer_2_nodes, layer_3_nodes]))
layer_3_bias = tf.Variable(tf.random_normal([layer_3_nodes]))
out_layer = tf.Variable(tf.random_normal([layer_3_nodes, output_nodes]))
out_layer_bias = tf.Variable(tf.random_normal([output_nodes]))
l1_output = tf.nn.relu(tf.matmul(network_input, layer_1) + layer_1_bias)
l2_output = tf.nn.relu(tf.matmul(l1_output, layer_2) + layer_2_bias)
l3_output = tf.nn.relu(tf.matmul(l2_output, layer_3) + layer_3_bias)
ntwk_output_1 = tf.matmul(l3_output, out_layer) + out_layer_bias
ntwk_output_2 = tf.nn.softmax(ntwk_output_1)
cf =
tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=ntwk_output_1,
labels=target_output))
ts = tf.train.GradientDescentOptimizer(learning_rate).minimize(cf)
cp = tf.equal(tf.argmax(ntwk_output_2, 1), tf.argmax(target_output, 1))
acc = tf.reduce_mean(tf.cast(cp, tf.float32))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
num_epochs = 10
for epoch in range(num_epochs):
total_cost = 0
for _ in range(int(mnist.train.num_examples / batch_size)):
batch_x, batch_y = mnist.train.next_batch(batch_size)
t, c = sess.run([ts, cf], feed_dict={network_input: batch_x, target_output: batch_y})
total_cost += c
print('Epoch', epoch, 'completed out of', num_epochs, 'loss:', total_cost)
print('Accuracy:', acc.eval({network_input: mnist.test.images,target_output: mnist.test.labels}))

with tf.Session() as sess:
number_prediction = tf.argmax(ntwk_output_2 , 1)
number_prediction = sess.run(number_prediction , feed_dict={network_input :
yourImageNdArray } )
print("your prediction : ",number_prediction)
What you need to know :
ntwk_ouput_2 is the ouput of the neural net which give you 10 probabilities - - you take the greatest one with tf.argmax ( tf argmax does not return the max value but the position of it )
sess.run is responsible of running your tensorflow graph and evaluating the tensor given in the first parameter
you need also to feed your network with the image you want to predict in feed_dict
Hope that helps!

The problem is you are not saving your model at any point of time during the training process.
You can do this during training:
ckpt_path = path_to_save_model
saver = tf.train.saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
num_epochs = 10
for epoch in range(num_epochs):
total_cost = 0
for _ in range(int(mnist.train.num_examples / batch_size)):
batch_x, batch_y =
mnist.train.next_batch(batch_size)
t, c = sess.run([ts, cf], feed_dict=
{network_input: batch_x, target_output: batch_y})
total_cost += c
print('Epoch', epoch, 'completed out of',
num_epochs, 'loss:', total_cost)
if (epoch+1)%10 == 0:
saver.saver(sess, ckpt_path)
print('Accuracy:', acc.eval({network_input:
mnist.test.images,target_output:
mnist.test.labels}))
For running the trained model you can do the following:
with tf.Session() as sess:
meta_graph = [ i for i in os.listdir(ckpt_path) if i.endswith('.meta')]
tf.train.import_meta_graph(os.path.join(checkpoint_path, meta_graph_path[0]))
saver = tf.train.Saver()
saver.restore(sess, ckpt_path)
#img = read your image here
pred = sess.run(ntwk_output_2, feed_dict={network_input: img}
output = np.argmax(pred)
For more reference you can follow this link

Incorrect value of placeholder

I am trying to execute the following code which is using MNIST dataset in Tensorflow, with images of shape 28 * 28 = 784 and 10 classes (0-9 digits) as output, I am getting an error that is showed as follows :
InvalidArgumentError: You must feed a value for placeholder tensor 'Placeholder_33' with dtype float and shape [?,10]
# Import MNIST data
#import input_data
#mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
import tensorflow as tf
# Set parameters
learning_rate = 0.01
training_iteration = 30
batch_size = 100
display_step = 2
# TF graph input
x = tf.placeholder("float", [None, 784]) # mnist data image of shape 28*28=784
y = tf.placeholder("float", [None, 10]) # 0-9 digits recognition => 10 classes
# Create a model
# Set model weights
W = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))
with tf.name_scope("Wx_b") as scope:
# Construct a linear model
model = tf.nn.softmax(tf.matmul(x, W) + b) # Softmax
# Add summary ops to collect data
w_h = tf.summary.histogram("weights", W)
b_h = tf.summary.histogram("biases", b)
# More name scopes will clean up graph representation
with tf.name_scope("cost_function") as scope:
# Minimize error using cross entropy
# Cross entropy
cost_function = -tf.reduce_sum(y*tf.log(model))
# Create a summary to monitor the cost function
tf.summary.scalar("cost_function", cost_function)
with tf.name_scope("train") as scope:
# Gradient descent
optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost_function)
# Initializing the variables
init = tf.initialize_all_variables()
# Merge all summaries into a single operator
merged_summary_op = tf.summary.merge_all()
# Launch the graph
with tf.Session() as sess:
sess.run(init)
summary_writer = tf.summary.FileWriter('/home/raed/Tensorflow/tensorflow_demo', graph_def=sess.graph_def)
# Training cycle
for iteration in range(training_iteration):
avg_cost = 0.
total_batch = int(mnist.train.num_examples/batch_size)
# Loop over all batches
for i in range(total_batch):
batch_xs, batch_ys = mnist.train.next_batch(batch_size)
# Fit training using batch data
sess.run(optimizer, feed_dict={x: batch_xs, y: batch_ys})
# Compute the average loss
avg_cost += sess.run(cost_function, feed_dict={x: batch_xs, y: batch_ys})/total_batch
# Write logs for each iteration
summary_str = sess.run(merged_summary_op, feed_dict={x: batch_xs, y: batch_ys})
summary_writer.add_summary(summary_str, iteration*total_batch + i)
# Display logs per iteration step
if iteration % display_step == 0:
print ("Iteration:" "%04d" % (iteration + 1), "cost=", "{:.9f}".format(avg_cost))
print ("Tuning completed!")
# Test the model
predictions = tf.equal(tf.argmax(model, 1), tf.argmax(y, 1))
# Calculate accuracy
accuracy = tf.reduce_mean(tf.cast(predictions, "float"))
print ("Accuracy:", accuracy.eval({x: mnist.test.images, y: mnist.test.labels}))

Restoring model take to long

I have a problem restring my model. I trained model and saved a model using this code. I'm not really sure if this is the proper method I would be grateful for suggestions. The problem occurs when I'm trying to restore model. I need it only to predict, it won't be furder trained. It takes forever to restore parameters from the model. How can I improve my model saver or model restorer to make it quick, under the assumption I need it only for predicting.
X = tf.placeholder(tf.float32, [None, 56, 56, 1])
Y_ = tf.placeholder(tf.float32, [None, 36])
L1 = 432
L2 = 72
L3 = 36
W1 = tf.Variable(tf.truncated_normal([3136, L1], stddev=0.1))
b1 = tf.Variable(tf.zeros([L1]))
W2 = tf.Variable(tf.truncated_normal([L1, L2], stddev=0.1))
b2 = tf.Variable(tf.zeros([L2]))
W3 = tf.Variable(tf.truncated_normal([L2, L3], stddev=0.1))
b3 = tf.Variable(tf.zeros([L3]))
XX = tf.reshape(X, [-1, 3136])
Y1 = tf.nn.sigmoid(tf.matmul(XX, W1) + b1)
Y1 = tf.nn.dropout(Y1, keep_prob=0.8)
Y2 = tf.nn.sigmoid(tf.matmul(Y1, W2) + b2)
Y2 = tf.nn.dropout(Y2, keep_prob=0.8)
Ylogits = tf.matmul(Y2, W3) + b3
Y = tf.nn.softmax(Ylogits)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits_v2(logits=Ylogits, labels=Y_)
cross_entropy = tf.reduce_mean(cross_entropy) * 100
correct_prediction = tf.equal(tf.argmax(Y, 1), tf.argmax(Y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
train_step = tf.train.GradientDescentOptimizer(0.0001).minimize(cross_entropy)
allweights = tf.concat([tf.reshape(W1, [-1]), tf.reshape(W2, [-1]), tf.reshape(W3, [-1])], 0)
allbiases = tf.concat([tf.reshape(b1, [-1]), tf.reshape(b2, [-1]), tf.reshape(b3, [-1])], 0)
init = tf.global_variables_initializer()
saver = tf.train.Saver()
def next_batch(x, y, batch, step):
x_temp = x[cur_step:(step+batch)]
y_temp = np.squeeze(y[step:(step + batch)])
return x_temp, y_temp
with tf.Session() as sess:
sess.run(init)
cur_step = 0
for i in range(NUM_ITERS + 1):
batch_X, batch_Y = next_batch(train_xx, train_yy, BATCH, cur_step)
if i % DISPLAY_STEP == 0:
acc_trn, loss_trn, w, b = sess.run([accuracy, cross_entropy, allweights, allbiases], feed_dict={X: batch_X, Y_: batch_Y})
acc_tst, loss_tst = sess.run([accuracy, cross_entropy], feed_dict={X: test_xx, Y_: test_yy})
sess.run(train_step, feed_dict={X: batch_X, Y_: batch_Y})
save_path = saver.save(sess, "abc/model")
Restore:
X = tf.placeholder(tf.float32, [None, 56, 56, 1])
Y_ = tf.placeholder(tf.float32, [None, 36])
L1 = 432
L2 = 72
L3 = 36
W1 = tf.Variable(tf.truncated_normal([3136, L1], stddev=0.1))
b1 = tf.Variable(tf.zeros([L1]))
W2 = tf.Variable(tf.truncated_normal([L1, L2], stddev=0.1))
b2 = tf.Variable(tf.zeros([L2]))
W3 = tf.Variable(tf.truncated_normal([L2, L3], stddev=0.1))
b3 = tf.Variable(tf.zeros([L3]))
XX = tf.reshape(X, [-1, 3136])
Y1 = tf.nn.sigmoid(tf.matmul(XX, W1) + b1)
Y1 = tf.nn.dropout(Y1, keep_prob=0.8)
Y2 = tf.nn.sigmoid(tf.matmul(Y1, W2) + b2)
Y2 = tf.nn.dropout(Y2, keep_prob=0.8)
Ylogits = tf.matmul(Y2, W3) + b3
Y = tf.nn.softmax(Ylogits)
with tf.Session() as sess:
saver = tf.train.Saver()
saver = tf.train.import_meta_graph('model.meta')
saver.restore(sess, 'model')
EDIT: Maybe a fact that model is trained using Google Colab's GPU and I'm restoring it on my PC is important.

Its a duplicate of: Tensorflow: how to save/restore a model?.
Your saving of the model is right but not your restoring. What you are doing is trying to create a new graph with the same nodes as the saved model, instead of restoring it from the saved graph. The following steps should address your issue on how to restore a model:
#Start with resetting the default graph
tf.reset_default_graph()
with tf.Session() as sess:
# Nodes:Before loading the graph
print([n.name for n in tf.get_default_graph().as_graph_def().node])
# Output is [] as no graph is loaded yet.
# First let's load meta graph
saver = tf.train.import_meta_graph("abc/model.meta")
# Nodes:after loading the graph'
print([n.name for n in tf.get_default_graph().as_graph_def().node])
# Output is [save/RestoreV2/shape_and_slices', 'save/RestoreV2/tensor_ ...]
# The above step doesnt load the weights, can be checked by
print(sess.run('Variable_1:0'))
# Error: attempting to use uninitialized graph.
#load the weights
saver.restore(sess,tf.train.latest_checkpoint('./abc/'))
print(sess.run('Variable_1:0'))
# Output: [-2.80421402e-04 3.53254407e-04 ...]
Now we have the nodes loaded and ready, you need to access some of them for inference. But since the nodes are not named properly, its not easy to figure out which node are inputs and outputs. To avoid this you need to name the tensors/ops when saving the model properly using the name argument like:
X = tf.placeholder(tf.float32, [None, 56, 56, 1], name='X')
Y = tf.identity(f.nn.softmax(Ylogits), name='logits').
In your inference graph once you have loaded the graph and weights, you can get these tensors using get_tensor_by_name:
with tf.Session() as sess:
#Load the graph and weights as above
....
graph = tf.get_default_graph()
X_infer = graph.get_tensor_by_name('X:0')
Y_infer = graph.get_tensor_by_name('logits:0')
sess.run(Y_infer,{X_infer:new_input}

How to print the predicted 'y' or 'output' matrix in this TensorFlow code?

I have a training model in TensorFlow (see below code).
My cumulative 'Test Accuracy' is showing: 0.92357 after training my model and I want to check it to be sure I didn't mess something up.
How do I print a predicted output matrix or 'y' after I train the model given the code I have below?
# x will be the input matrix flattened (28x29)
x = tf.placeholder(tf.float32, [None, 812])
# Define the weights (initial value doesn't matter since these will be learned)
W = tf.Variable(tf.random_uniform([812, 812], minval=0, dtype=tf.float32))
b = tf.Variable(tf.random_uniform([812], minval=0, dtype=tf.float32))
# Predict output matrix
y = tf.nn.softmax(tf.matmul(x, W) + b)
# Actual output matrix from the training set
y_ = tf.placeholder(tf.float32, [None, 812])
# Calculate loss and optimize
cross_entropy = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(labels=y_, logits=y))
train_step = tf.train.AdamOptimizer(0.025).minimize(cross_entropy)
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()
a, b = get_batch()
train_len = len(a)
correct_prediction = tf.equal(y_, y)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# Training
for i in range(train_len):
batch_xs = a[i]
batch_ys = b[i]
_, loss, acc = sess.run([train_step, cross_entropy, accuracy], feed_dict={x: batch_xs, y_: batch_ys})
print("Loss= " + "{:.6f}".format(loss) + " Accuracy= " + "{:.5f}".format(acc))
# Test trained model
cumulative_accuracy = 0.0
for i in range(train_len):
acc_batch_xs = a[i]
acc_batch_ys = b[i]
cumulative_accuracy += accuracy.eval(feed_dict={x: acc_batch_xs, y_: acc_batch_ys})
print("Test Accuracy= {}".format(cumulative_accuracy / train_len))

The value of any tensor object can be obtained by using
tensorFlowObject.eval()
Hence, you can use y.eval() to get the value of y

Function approximation with Neural Network - Loss 0

I'm trying to create a NN to approximate functions (sine, cos, custom...) but I'm struggling with the format, I don't want to use input-label, but rather, input-output. How do I change it?
I'm following this tutorial
import tensorflow as tf
import random
from math import sin
import numpy as np
n_nodes_hl1 = 500
n_nodes_hl2 = 500
n_nodes_hl3 = 500
n_inputs = 1 # CHANGES HERE
n_outputs = 1 #CHANGES HERE
batch_size = 100
x = tf.placeholder('float', [None, n_inputs]) #CHANGES HERE
y = tf.placeholder('float', [None, n_outputs]) #CHANGES HERE
def neural_network_model(data):
hidden_layer_1 = {'weights':tf.Variable(tf.random_normal([n_inputs, n_nodes_hl1])),
'biases': tf.Variable(tf.random_normal([n_nodes_hl1]))} #CHANGES HERE
hidden_layer_2 = {'weights':tf.Variable(tf.random_normal([n_nodes_hl1, n_nodes_hl2])),
'biases': tf.Variable(tf.random_normal([n_nodes_hl2]))}
hidden_layer_3 = {'weights':tf.Variable(tf.random_normal([n_nodes_hl2, n_nodes_hl3])),
'biases': tf.Variable(tf.random_normal([n_nodes_hl2]))}
output_layer = {'weights':tf.Variable(tf.random_normal([n_nodes_hl3, n_outputs])),
'biases': tf.Variable(tf.random_normal([n_outputs]))} #CHANGES HERE
l1 = tf.add(tf.matmul(data, hidden_layer_1['weights']), hidden_layer_1['biases'])
l1 = tf.nn.relu(l1)
l2 = tf.add(tf.matmul(l1, hidden_layer_2['weights']), hidden_layer_2['biases'])
l2 = tf.nn.relu(l2)
l3 = tf.add(tf.matmul(l2, hidden_layer_3['weights']), hidden_layer_3['biases'])
l3 = tf.nn.relu(l3)
output = tf.add(tf.matmul(l3, output_layer['weights']), output_layer['biases'])
return output
def train_neural_network(x):
prediction = neural_network_model(x)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=y))
optmizer = tf.train.AdamOptimizer().minimize(cost)
epochs = 10
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for epoch in range(epochs):
loss = 0
for _ in range(batch_size^2): #CHANGES HERE
batch_x, batch_y = generate_input_output(batch_size) #CHANGES HERE
a, c = sess.run([optmizer, cost], feed_dict={x: batch_x, y:batch_y})
loss += c
print("Epoch:", epoch+1, "out of", epochs, "- Loss:", loss)
correct = tf.equal(tf.argmax(prediction, 1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct, 'float'))
test_x, test_y = generate_input_output(batch_size) #CHANGES HERE
print('Accuracy', accuracy.eval({x:test_x, y:test_y}))
def desired_function(x): #CHANGES HERE
return sin(x)
def generate_input_output(batch_size): #CHANGES HERE
batch_x = [random.uniform(-10, 10) for _ in range(batch_size)]
batch_y = [desired_function(x) for x in batch_x]
batch_x = np.reshape(batch_x, (-1, 1))
batch_y = np.reshape(batch_y, (-1, 1))
return batch_x, batch_y
train_neural_network(x)

Your solution seems very verbose to me. I'll post a much simplified solution, could you point out what part of it you don't understand?
import tensorflow as tf
import numpy as np
n_nodes_hl1 = 500
n_nodes_hl2 = 500
n_nodes_hl3 = 500
batch_size = 100
x = tf.placeholder('float', [None, 1])
y = tf.placeholder('float', [None, 1])
x1 = tf.contrib.layers.fully_connected(x, n_nodes_hl1)
x2 = tf.contrib.layers.fully_connected(x1, n_nodes_hl2)
x3 = tf.contrib.layers.fully_connected(x2, n_nodes_hl3)
result = tf.contrib.layers.fully_connected(x3, 1,
activation_fn=None)
loss = tf.nn.l2_loss(result - y)
train_op = tf.train.AdamOptimizer().minimize(loss)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for i in range(10000):
xpts = np.random.rand(batch_size) * 10
ypts = np.sin(xpts)
_, loss_result = sess.run([train_op, loss],
feed_dict={x: xpts[:, None],
y: ypts[:, None]})
print('iteration {}, loss={}'.format(i, loss_result))
It seems to me that your solution was intended for classification and you have not fully rewrote it for regression, since there are things like softmax_cross_entropy_with_logits left in there, which you certainly don't want in a regression network.

Have not tried it myself, so there might be more things you need to change to get the model to run, but you will definitely want to change this line:
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=y))
to something more like:
cost = tf.reduce_sum(tf.square(prediction - y))
Basically, your cost function is much simpler in this case...you just want to reduce the sum of the squared difference between the network's output and the expected y value.