We Keep Coding

how can I calculate recall, precision and f-score for this model?

I've been working on implementing this model for months now and finally get it to work. Now I was looking to calculate metrics for this model like F-score, Recall, Precision etc. I looked at examples and they do it by splitting data which I've not been able to implement. Can I calculate it like model loss and total loss is calculated here???
from nets import model
from utils.data_provider import data_provider
FLAGS = tf.app.flags.FLAGS
gpus = list(range(len(FLAGS.gpu_list.split(','))))
logger.setLevel(cfg.debug)
def tower_loss(images, seg_maps_gt, training_masks, reuse_variables=None):
# Build inference graph
with tf.variable_scope(tf.get_variable_scope(), reuse=reuse_variables):
seg_maps_pred = model.model(images, is_training=True)
model_loss = model.loss(seg_maps_gt, seg_maps_pred, training_masks)
total_loss = tf.add_n([model_loss] + tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
# add summary
if reuse_variables is None:
tf.summary.image('input', images)
tf.summary.image('seg_map_0_gt', seg_maps_gt[:, :, :, 0:1] * 255)
tf.summary.image('seg_map_0_pred', seg_maps_pred[:, :, :, 0:1] * 255)
tf.summary.image('training_masks', training_masks)
tf.summary.scalar('model_loss', model_loss)
tf.summary.scalar('total_loss', total_loss)
return total_loss, model_loss
def average_gradients(tower_grads):
average_grads = []
for grad_and_vars in zip(*tower_grads):
grads = []
for g, _ in grad_and_vars:
expanded_g = tf.expand_dims(g, 0)
grads.append(expanded_g)
grad = tf.concat(grads, 0)
grad = tf.reduce_mean(grad, 0)
v = grad_and_vars[0][1]
grad_and_var = (grad, v)
average_grads.append(grad_and_var)
return average_grads
def main(argv=None):
import os
os.environ['CUDA_VISIBLE_DEVICES'] = FLAGS.gpu_list
if not tf.gfile.Exists(FLAGS.checkpoint_path):
tf.gfile.MkDir(FLAGS.checkpoint_path)
else:
if not FLAGS.restore:
tf.gfile.DeleteRecursively(FLAGS.checkpoint_path)
tf.gfile.MkDir(FLAGS.checkpoint_path)
input_images = tf.placeholder(tf.float32, shape=[None, None, None, 3], name='input_images')
input_seg_maps = tf.placeholder(tf.float32, shape=[None, None, None, 6], name='input_score_maps')
input_training_masks = tf.placeholder(tf.float32, shape=[None, None, None, 1], name='input_training_masks')
global_step = tf.get_variable('global_step', [], initializer=tf.constant_initializer(0), trainable=False)
learning_rate = tf.train.exponential_decay(FLAGS.learning_rate, global_step, decay_steps=10000, decay_rate=0.94, staircase=True)
# add summary
tf.summary.scalar('learning_rate', learning_rate)
opt = tf.train.AdamOptimizer(learning_rate)
# split
input_images_split = tf.split(input_images, len(gpus))
input_seg_maps_split = tf.split(input_seg_maps, len(gpus))
input_training_masks_split = tf.split(input_training_masks, len(gpus))
tower_grads = []
reuse_variables = None
for i, gpu_id in enumerate(gpus):
with tf.device('/gpu:%d' % gpu_id):
with tf.name_scope('model_%d' % gpu_id) as scope:
iis = input_images_split[i]
isegs = input_seg_maps_split[i]
itms = input_training_masks_split[i]
total_loss, model_loss = tower_loss(iis, isegs, itms, reuse_variables)
batch_norm_updates_op = tf.group(*tf.get_collection(tf.GraphKeys.UPDATE_OPS, scope))
reuse_variables = True
grads = opt.compute_gradients(total_loss)
tower_grads.append(grads)
grads = average_gradients(tower_grads)
apply_gradient_op = opt.apply_gradients(grads, global_step=global_step)
summary_op = tf.summary.merge_all()
# save moving average
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
variables_averages_op = variable_averages.apply(tf.trainable_variables())
# batch norm updates
with tf.control_dependencies([variables_averages_op, apply_gradient_op, batch_norm_updates_op]):
train_op = tf.no_op(name='train_op')
saver = tf.train.Saver(tf.global_variables())
summary_writer = tf.summary.FileWriter(FLAGS.checkpoint_path, tf.get_default_graph())
init = tf.global_variables_initializer()
if FLAGS.pretrained_model_path is not None:
variable_restore_op = slim.assign_from_checkpoint_fn(FLAGS.pretrained_model_path, slim.get_trainable_variables(),
ignore_missing_vars=True)
gpu_options=tf.GPUOptions(allow_growth=True)
#gpu_options=tf.GPUOptions(per_process_gpu_memory_fraction=0.75)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options, allow_soft_placement=True)) as sess:
if FLAGS.restore:
logger.info('continue training from previous checkpoint')
ckpt = tf.train.latest_checkpoint(FLAGS.checkpoint_path)
logger.debug(ckpt)
saver.restore(sess, ckpt)
else:
sess.run(init)
if FLAGS.pretrained_model_path is not None:
variable_restore_op(sess)
data_generator = data_provider.get_batch(num_workers=FLAGS.num_readers,
input_size=FLAGS.input_size,
batch_size=FLAGS.batch_size_per_gpu * len(gpus))
start = time.time()
for step in range(FLAGS.max_steps):
data = next(data_generator)
ml, tl, _ = sess.run([model_loss, total_loss, train_op], feed_dict={input_images: data[0],
input_seg_maps: data[2],
input_training_masks: data[3]})
if np.isnan(tl):
logger.error('Loss diverged, stop training')
break
if step % 10 == 0:
avg_time_per_step = (time.time() - start)/10
avg_examples_per_second = (10 * FLAGS.batch_size_per_gpu * len(gpus))/(time.time() - start)
start = time.time()
logger.info('Step {:06d}, model loss {:.4f}, total loss {:.4f}, {:.2f} seconds/step, {:.2f} examples/second'.format(
step, ml, tl, avg_time_per_step, avg_examples_per_second))
if step % FLAGS.save_checkpoint_steps == 0:
saver.save(sess, os.path.join(FLAGS.checkpoint_path, 'model.ckpt'), global_step=global_step)
if step % FLAGS.save_summary_steps == 0:
_, tl, summary_str = sess.run([train_op, total_loss, summary_op], feed_dict={input_images: data[0],
input_seg_maps: data[2],
input_training_masks: data[3]})
summary_writer.add_summary(summary_str, global_step=step)
Sorry for posting bunch of code lines but I really have no idea how to calculate metrics from all this?

Scikit-learn has a great API for calculating precision, recall and F1 score for each class:
https://scikit-learn.org/stable/modules/generated/sklearn.metrics.classification_report.html
Y_pred = sess.run([nn_output], feed_dict={input_images: input_data[0], input_seg_maps: input_data[2], input_training_masks: input_data[3]]}) # Feed according to your model
ClassificationReport = sklearn.metrics.classification_report(Y_pred, Y_true, output_dict=True)

Tensorflow save/load frozen tf.graph and run classification on loaded graph

as I have mentioned in the topic I would like to save my tf.graph into a frozen_graph.pb file. This should save space later I will try to run it an a jetson tx2. I have made a short MNIST example describing my problem. I run tf 1.7 on python 3.5.
Question1: As far as I understood my freeze_graph method takes a checkpoint file transfers all variables to constants except the ones i define with the second parameter. When I try to get the correct tensorname I wrote loggits.name but I get an error no Tensor with that name found in graph.
Question2: After that I would be able to extract a frozen graph, how can i load it back and run an classification on that.
My Code is attached and should work in a single py file.
Thank you very much in advance
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
from tensorflow.examples.tutorials.mnist import input_data
import os
import time
import tensorflow as tf
import os
import argparse
#METHODS I WANT TO TEST
#TAKE THE CHECKPOINT FILE AND DELETE ALL NOTES THAT ARE NOT USEFUL
def freeze_graph(checkpoint_directory,output_node_names):
#checkpoint = tf.train.get_checkpoint_state(checkpoint_directory)
print(checkpoint_directory)
checkpoint = tf.train.get_checkpoint_state(checkpoint_directory)
input_checkpoint = checkpoint.model_checkpoint_path
absolute_model_dir = str(os.sep).join(input_checkpoint.split(os.sep)[:-1])
output_graph = absolute_model_dir + "/frozen_model.pb"
clear_devices = True
with tf.Session(graph = tf.Graph()) as sess:
#import the metagraph in default graph
saver = tf.train.import_meta_graph(input_checkpoint + '.meta',clear_devices=clear_devices)
#restore the weights
saver.restore(sess,input_checkpoint)
#wrap variables to constants
[print(n.name) for n in tf.get_default_graph().as_graph_def().node]
output_graph_def = tf.graph_util.convert_variables_to_constants(sess, tf.get_default_graph().as_graph_def(),output_node_names.split(","))
with tf.gfile.GFile(output_graph, "wb") as f:
f.write(output_graph_def.SerializeToString())
print("%d ops in the final graph." %len(output_graph_def.node))
return output_graph_def
#HERE IS THE METHOD THAT ALLOWS ME TO LOAD MY FROZEN GRAPH AS GRAPH
def load_graph(frozen_graph_filename):
with tf.gfile.GFile(frozen_graph_filename,"rb") as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
with tf.Graph().as_default() as graph:
tf.import_graph_def(graph_def, name = "prefix")
return graph
#get the data
mnist = input_data.read_data_sets("MNIST_data/",one_hot=True)
#NETWORK PARAMETERS
learning_rate = 0.01
dropout = 0.75
display_step = 1
filter_height = 5
filter_width = 5
depth_in = 1
depth_out1 = 64
depth_out2 = 128
#PARAMETERS OF THE DATASET
input_height = 28
input_width = 28
n_classes = 10
#TRAINING PARAMETERS
epochs = 1
batch_size = 256
num_batches = int(mnist.train.num_examples/batch_size)
x = tf.placeholder(tf.float32,[None,28*28],name = "input")
y = tf.placeholder(tf.float32,[None,n_classes])
keep_prob = tf.placeholder(tf.float32)
weights = {'wc1': tf.Variable(tf.random_normal([filter_height,filter_width,depth_in,depth_out1])),
'wc2': tf.Variable(tf.random_normal([filter_height, filter_width, depth_out1, depth_out2])),
'wd1': tf.Variable(tf.random_normal([int(input_height/4)*int(input_height/4)*depth_out2,1024])),
'out': tf.Variable(tf.random_normal([1024,n_classes]))}
biases = {'bc1': tf.Variable(tf.random_normal([depth_out1])),
'bc2': tf.Variable(tf.random_normal([depth_out2])),
'bd1': tf.Variable(tf.random_normal([1024])),
'out': tf.Variable(tf.random_normal([n_classes]))}
#DEFINE YOUR NEURAL NETWORKS LAYER OPERATIONS
def ops_conv2d(x,W,b,strides = 1, add_bias = True, activation = tf.nn.relu, use_activation = True):
x = tf.nn.conv2d(x,W,strides = [1,strides,strides,1],padding = 'SAME')
x = tf.nn.bias_add(x,b)
if use_activation:
return activation(x)
else:
return x
def ops_maxpool2d(x,stride=2):
return tf.nn.max_pool(x,ksize=[1,stride,stride,1],strides = [1,stride,stride,1], padding = 'SAME' )
def ops_dropout(input_fully_connected,dropout):
return tf.nn.dropout(input_fully_connected,dropout)
def ops_fullyconnected(input, activation = tf.nn.relu, use_activation = True):
fc = tf.reshape(input,[-1,weights['wd1'].get_shape().as_list()[0]])
fc = tf.add(tf.matmul(fc,weights['wd1']),biases['bd1'])
if use_activation:
return activation(fc)
else:
return fc
#DEFINE NETWORK ARCHTEKTURE (FORWARDPASS)
def build_network(x,weights,biases,dropout):
x = tf.reshape(x,shape=(-1,28,28,1))
conv_layer_1 = ops_conv2d(x,weights['wc1'],biases['bc1'],activation=tf.nn.relu, use_activation=True)
conv_layer_1 = ops_maxpool2d(conv_layer_1,2)
conv_layer_2 = ops_conv2d(conv_layer_1,weights['wc2'],biases['bc2'],activation=tf.nn.relu, use_activation=True)
conv_layer_2 = ops_maxpool2d(conv_layer_2,2)
fc1 = ops_fullyconnected(conv_layer_2, activation=tf.nn.relu, use_activation=True)
fc1 = ops_dropout(fc1,dropout)
logits = tf.add(tf.matmul(fc1,weights['out']),biases['out'],name = "logits")
return logits
#DEFINE TENSORFLOW BACKPROPAGATION OBJECTS (BACKWARDPASS)
logits = build_network(x,weights,biases,keep_prob)
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = logits,labels = y))
#CHOSE AN OPTIMIZER
optimizer = tf.train.AdadeltaOptimizer(learning_rate=learning_rate).minimize(loss=loss)
predicted_labels = tf.equal(tf.argmax(logits,1),tf.argmax(y,1))
#EVALUATION PARAMETERS
acc = tf.reduce_mean(tf.cast(predicted_labels,tf.float32))
#NOW INITIALIZE ALL TF VARIABLES
init = tf.global_variables_initializer()
saver = tf.train.Saver(max_to_keep=10)
#NOW START THE SESSION AND EXECUTE THE GRAPH
with tf.Session() as sess:
sess.run(init)
for i in range(epochs):
save_path = saver.save(sess, os.curdir + "checkpoints/MNIST_TEST.ckpt")
for j in range(num_batches):
batch_x, batch_y = mnist.train.next_batch(batch_size)
sess.run(optimizer, feed_dict={x:batch_x,y:batch_y,keep_prob:dropout})
losses,accs = sess.run([loss,acc],feed_dict={x:batch_x,y:batch_y,keep_prob:1.})
if epochs % display_step == 0:
print("EPOCH:",'%04d' % (i+1),
"loss =", "{:.9f}".format(losses),
"acc =", "{:.5f}".format(accs))
print("TRAINING COMPLETED")
#START PREDICTIONS
predicted_label = sess.run(logits,feed_dict={x:mnist.test.images[:256],keep_prob:1.})
test_classes = np.argmax(predicted_label,1)
print("TEST ACCURACY:",sess.run(acc,feed_dict={x:mnist.test.images[:256], y:mnist.test.labels[:256],keep_prob:1.}))
f,a = plt.subplots(1,10,figsize = (10,2))
for i in range(10):
a[i].imshow(np.reshape(mnist.test.images[i],(28,28)))
print( test_classes[i])
print("TOTAL EXAMPLE FINNISHED")
freeze_graph(os.curdir + "checkpoints" + os.sep, logits.name)
graph = load_graph(os.curdir + os.sep + "checkpoints" + os.sep + "frozen_model.pb")
with tf.Session(graph) as sess:
sess.run(init)
predicted_label = sess.run(logits, feed_dict={x: mnist.test.images[:256], keep_prob: 1.})
print(predicted_label)

if anybody has the same problem here is a description how i solved it.
Saving and Loading the data:
First of all note that I have now a different pipeline. First of all I save the session in a saver (ckpt files). Afterwards I construct a metagaph (graph.pb). This graph is then transfered into a frozen graph (frozen.pb). To load the frozen graph I use the load_frozen_graph_from_session method. Inside that method I also test a forward pass through my network.
Running an inference on the loaded graph:
First I name my tensors x (name = "input") this will result in a tensorname ("input:0")
so when you try to fill this placeholder in the new session you need predicted_label = sess.run("output:0", feed_dict={"input:0":mnist.test.images[:256], "keep_prob:0": 1.})
The output is the logit and not the prediction inside my network. This is because if you run the session it will run until it hits the variable you want to fetch. would I take the prediction I need also the placeholder for my y (name=label).
Here is the full code:
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
from tensorflow.examples.tutorials.mnist import input_data
import os
import time
import tensorflow as tf
import os
import argparse
from tensorflow.python.platform import gfile
from tensorflow.python.framework.graph_util import convert_variables_to_constants
#METHODS I WANT TO TEST
def freeze_graph_from_Session(sess,saver):
# convert_variables_to_constants(sess, input_graph_def, output_node_names, variable_names_whitelist=None)
save_graph(sess,saver)
with gfile.FastGFile("./tmp/" + "graph.pb", 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
frozen_graph_def = convert_variables_to_constants(sess, graph_def, ["output"])
with tf.gfile.GFile("./tmp/" + "frozen.pb", "wb") as f:
f.write(frozen_graph_def.SerializeToString())
def save_graph(sess, saver):
saver.save(sess, "./tmp/model", write_meta_graph=True, global_step=1)
with open("./tmp/" + "graph.pb", 'wb') as f:
f.write(sess.graph_def.SerializeToString())
#sess.close()
def load_frozen_graph_from_session():
filename = "./tmp/" + "frozen.pb"
print("LOADING GRAPH")
with tf.gfile.GFile(filename, "rb") as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
print("OPEN GRAPH")
with tf.Graph().as_default() as graph:
print("DEFINE INPUT")
new_input = tf.placeholder(tf.float32, [None, 28 * 28], name="new_input")
print("DEFINE INPUT MAP")
tf.import_graph_def(
graph_def,
# usually, during training you use queues, but at inference time use placeholders
# this turns into "input
input_map={"input:0": new_input},
return_elements=None,
# if input_map is not None, needs a name
name="bla",
op_dict=None,
producer_op_list=None
)
checkpoint_path = tf.train.latest_checkpoint("./tmp/")
with tf.Session(graph=graph) as sess:
saver = tf.train.import_meta_graph(checkpoint_path + ".meta", import_scope=None)
saver.restore(sess, checkpoint_path)
print("TRY FORWARD RUN THROUGH LOADED GRAPH")
predicted_label = sess.run("output:0", feed_dict={"input:0":mnist.test.images[:256], "keep_prob:0": 1.})
print("output", predicted_label)
f, a = plt.subplots(1, 10, figsize=(10, 2))
test_classes = np.argmax(predicted_label, 1)
for i in range(10):
a[i].imshow(np.reshape(mnist.test.images[i], (28, 28)))
print(test_classes[i])
print ("output:", test_classes)
#TAKE THE CHECKPOINT FILE AND DELETE ALL NOTES THAT ARE NOT USEFUL
def freeze_graph(checkpoint_directory,output_node_names):
#checkpoint = tf.train.get_checkpoint_state(checkpoint_directory)
print(checkpoint_directory)
checkpoint = tf.train.get_checkpoint_state(checkpoint_directory)
input_checkpoint = checkpoint.model_checkpoint_path
absolute_model_dir = str(os.sep).join(input_checkpoint.split(os.sep)[:-1])
output_graph = absolute_model_dir + "/frozen_model.pb"
clear_devices = True
with tf.Session(graph = tf.Graph()) as sess:
#import the metagraph in default graph
saver = tf.train.import_meta_graph(input_checkpoint + '.meta',clear_devices=clear_devices)
#restore the weights
saver.restore(sess,input_checkpoint)
#wrap variables to constants
[print(n.name) for n in tf.get_default_graph().as_graph_def().node]
output_graph_def = tf.graph_util.convert_variables_to_constants(sess, tf.get_default_graph().as_graph_def(),output_node_names.split(","))
with tf.gfile.GFile(output_graph, "wb") as f:
f.write(output_graph_def.SerializeToString())
print("%d ops in the final graph." %len(output_graph_def.node))
return output_graph_def
#HERE IS THE METHOD THAT ALLOWS ME TO LOAD MY FROZEN GRAPH AS GRAPH
def load_graph(frozen_graph_filename):
with tf.gfile.GFile(frozen_graph_filename,"rb") as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
with tf.Graph().as_default() as graph:
tf.import_graph_def(graph_def, name = "prefix")
return graph
#get the data
mnist = input_data.read_data_sets("MNIST_data/",one_hot=True)
print(mnist.test.labels[:256])
print("load_freeze_graph_from_session: STARTED")
load_frozen_graph_from_session()
print("load_freeze_graph_from_session: ENDED")
exit()
#NETWORK PARAMETERS
learning_rate = 0.01
dropout = 0.75
display_step = 1
filter_height = 5
filter_width = 5
depth_in = 1
depth_out1 = 64
depth_out2 = 128
#PARAMETERS OF THE DATASET
input_height = 28
input_width = 28
n_classes = 10
#TRAINING PARAMETERS
epochs = 1
batch_size = 256
num_batches = int(mnist.train.num_examples/batch_size)
x = tf.placeholder(tf.float32,[None,28*28],name="input")
y = tf.placeholder(tf.float32,[None,n_classes],name = "label")
keep_prob = tf.placeholder(tf.float32,name = "keep_prob")
weights = {'wc1': tf.Variable(tf.random_normal([filter_height,filter_width,depth_in,depth_out1])),
'wc2': tf.Variable(tf.random_normal([filter_height, filter_width, depth_out1, depth_out2])),
'wd1': tf.Variable(tf.random_normal([int(input_height/4)*int(input_height/4)*depth_out2,1024])),
'out': tf.Variable(tf.random_normal([1024,n_classes]))}
biases = {'bc1': tf.Variable(tf.random_normal([depth_out1])),
'bc2': tf.Variable(tf.random_normal([depth_out2])),
'bd1': tf.Variable(tf.random_normal([1024])),
'out': tf.Variable(tf.random_normal([n_classes]))}
#DEFINE YOUR NEURAL NETWORKS LAYER OPERATIONS
def ops_conv2d(x,W,b,strides = 1, add_bias = True, activation = tf.nn.relu, use_activation = True):
x = tf.nn.conv2d(x,W,strides = [1,strides,strides,1],padding = 'SAME')
x = tf.nn.bias_add(x,b)
if use_activation:
return activation(x)
else:
return x
def ops_maxpool2d(x,stride=2):
return tf.nn.max_pool(x,ksize=[1,stride,stride,1],strides = [1,stride,stride,1], padding = 'SAME' )
def ops_dropout(input_fully_connected,dropout):
return tf.nn.dropout(input_fully_connected,dropout)
def ops_fullyconnected(input, activation = tf.nn.relu, use_activation = True):
fc = tf.reshape(input,[-1,weights['wd1'].get_shape().as_list()[0]])
fc = tf.add(tf.matmul(fc,weights['wd1']),biases['bd1'])
if use_activation:
return activation(fc)
else:
return fc
#DEFINE NETWORK ARCHTEKTURE (FORWARDPASS)
def build_network(x,weights,biases,dropout):
x = tf.reshape(x,shape=(-1,28,28,1))
conv_layer_1 = ops_conv2d(x,weights['wc1'],biases['bc1'],activation=tf.nn.relu, use_activation=True)
conv_layer_1 = ops_maxpool2d(conv_layer_1,2)
conv_layer_2 = ops_conv2d(conv_layer_1,weights['wc2'],biases['bc2'],activation=tf.nn.relu, use_activation=True)
conv_layer_2 = ops_maxpool2d(conv_layer_2,2)
fc1 = ops_fullyconnected(conv_layer_2, activation=tf.nn.relu, use_activation=True)
fc1 = ops_dropout(fc1,dropout)
logits = tf.add(tf.matmul(fc1,weights['out']),biases['out'],name = "output")
return logits
#DEFINE TENSORFLOW BACKPROPAGATION OBJECTS (BACKWARDPASS)
logits = build_network(x,weights,biases,keep_prob)
#freeze_graph(os.curdir + "checkpoints" + os.sep, logits.name)
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = logits,labels = y))
#CHOSE AN OPTIMIZER
optimizer = tf.train.AdadeltaOptimizer(learning_rate=learning_rate).minimize(loss=loss)
predicted_labels = tf.equal(tf.argmax(logits,1),tf.argmax(y,1))
#EVALUATION PARAMETERS
acc = tf.reduce_mean(tf.cast(predicted_labels,tf.float32))
#NOW INITIALIZE ALL TF VARIABLES
init = tf.global_variables_initializer()
saver = tf.train.Saver(max_to_keep=10)
#NOW START THE SESSION AND EXECUTE THE GRAPH
with tf.Session() as sess:
sess.run(init)
for i in range(epochs):
save_path = saver.save(sess, os.curdir + "checkpoints/MNIST_TEST.ckpt")
for j in range(num_batches):
batch_x, batch_y = mnist.train.next_batch(batch_size)
sess.run(optimizer, feed_dict={x:batch_x,y:batch_y,keep_prob:dropout})
losses,accs = sess.run([loss,acc],feed_dict={x:batch_x,y:batch_y,keep_prob:1.})
if epochs % display_step == 0:
print("EPOCH:",'%04d' % (i+1),
"loss =", "{:.9f}".format(losses),
"acc =", "{:.5f}".format(accs))
print("TRAINING COMPLETED")
#START PREDICTIONS
predicted_label = sess.run(logits,feed_dict={x:mnist.test.images[:256],keep_prob:1.})
test_classes = np.argmax(predicted_label,1)
print("TEST ACCURACY:",sess.run(acc,feed_dict={x:mnist.test.images[:256], y:mnist.test.labels[:256],keep_prob:1.}))
f,a = plt.subplots(1,10,figsize = (10,2))
for i in range(10):
a[i].imshow(np.reshape(mnist.test.images[i],(28,28)))
print( test_classes[i])
print("TOTAL EXAMPLE FINNISHED")
#freeze_graph(os.curdir + "checkpoints"+os.sep,logits)
print("freeze_graph_from_session: STARTED")
freeze_graph_from_Session(sess,saver)
print("freeze_graph_from_session: ENDED")
print("load_freeze_graph_from_session: STARTED")
load_frozen_graph_from_session()
print("load_freeze_graph_from_session: ENDED")
#with tf.Session() as sess:
#
# sess.run(init)
# graph = load_graph(os.curdir + os.sep + "checkpoints" + os.sep + "frozen_model.pb")
# predicted_label = sess.run(logits, feed_dict={x: mnist.test.images[:256], keep_prob: 1.})
# print(predicted_label)
Thanks goes out to my self. :)

Training CNN model by using activation function Selu

I am training my own model using Tensorflow. However, I got some trouble when I change my activation function from Relu to Selu.
This is what happened. Learning curve drops accidentally and I have no idea about what's going on.
my learning curve
like this.
For what I have known, Selu can prevent overfitting, so I try to implement it in my model. Is there any tips, or any condition when I want to use Selu?
This is my code:
this is the place where I change my activation function
-----
def conv2d_maxpool(x_tensor, conv_num_outputs, conv_ksize, conv_strides, pool_ksize, pool_strides, layer_name):
conv_layer = tf.layers.conv2d(x_tensor, conv_num_outputs, kernel_size=conv_ksize, strides=conv_strides, activation=tf.nn.selu, name = layer_name)
conv_layer = tf.layers.max_pooling2d(conv_layer, pool_size=pool_ksize, strides=pool_strides)
return conv_layer
-----
graph
tf.reset_default_graph()
#### placeholder ####
input_img = tf.placeholder(dtype=tf.float32, shape=(None, img_size, img_size, 3))
y_true = tf.placeholder(dtype=tf.float32, shape=(None, num_class))
keep_prob = tf.placeholder(tf.float32, name="keep_prob")
lr_in = tf.placeholder(dtype = tf.float32, name = 'learning_rate')
conv_ksize = (3,3)
conv_strides = (1,1)
pool_ksize = (2,2)
pool_strides = (2,2)
n_filters_1 = 32
n_filters_2 = 64
n_filters_3 = 128
n_filters_4 = 256
onebyone_ksize = (1,1)
#CNN
conv_1 = conv2d_maxpool(input_img, n_filters_1, conv_ksize, conv_strides, pool_ksize, pool_strides, layer_name = "conv1")
# conv_1 = tf.layers.conv2d(conv_1, conv_num_outputs, kernel_size=conv_ksize, strides=conv_strides, activation=tf.nn.relu)
# conv_1_norm = tf.layers.batch_normalization(conv_1, name = "batch_norm1")
# conv_1_dropout = tf.layers.dropout(conv_1_norm, rate = keep_prob, training = True, name = "dropout1")
conv_2 = conv2d_maxpool(conv_1, n_filters_2, conv_ksize, conv_strides, pool_ksize, pool_strides, layer_name = "conv2")
# conv_2_norm = tf.layers.batch_normalization(conv_2)
conv_3 = conv2d_maxpool(conv_2, n_filters_3, conv_ksize, conv_strides, pool_ksize, pool_strides, layer_name = "conv3")
# conv_3_norm = tf.layers.batch_normalization(conv_3, name = "batch_norm3")
# conv_3_dropout = tf.layers.dropout(conv_3_norm, rate = keep_prob, training = True, name = "dropout3")
conv_4 = conv2d_maxpool(conv_3, n_filters_4, conv_ksize, conv_strides, pool_ksize, pool_strides, layer_name = "conv4")
flatten = tf.layers.flatten(conv_4)
fc1 = tf.layers.dense(flatten, 256, activation = tf.nn.relu)
out = tf.layers.dense(fc1, 6, activation=None, name= "logits") #logit
predict = tf.nn.softmax(out)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = out, labels = y_true))
optimizer = tf.train.AdamOptimizer(lr).minimize(cost)
##accuracy
correct_pred = tf.equal(tf.argmax(out, 1), tf.argmax(y_true, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32), name='accuracy')
Training
#history/record
train_loss, train_acc = [], []
valid_loss, valid_acc = [], []
update_per_epoch = int(np.floor(X_train.shape[0] / batch_size))
## early stopping and learning rate congig
es_patience = 10
es_n = 0
lr_patience = 3
lr_n = 0
save_model_path = './save'
saver = tf.train.Saver()
sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)
# Initializing the variables
batch_gen = img_gen.flow(generator_input(X_train), y_train, batch_size = 32)
val_batch_gen = img_gen.flow(generator_input(X_valid), y_valid, batch_size = len(X_valid))
for i in range(epoch):
epoch_loss = 0
epoch_acc = 0
for j in range(update_per_epoch):
image, label = next(batch_gen)
_, this_loss, this_acc = sess.run([optimizer, cost, accuracy], feed_dict={
input_img : image,
y_true : label,
lr_in: lr,
keep_prob : keep_probability
})
epoch_loss += this_loss
epoch_acc += this_acc
## end of epoch
epoch_loss /= update_per_epoch
epoch_acc /= update_per_epoch
train_loss.append(epoch_loss)
train_acc.append(epoch_acc)
print('Epoch {:>2} Loss: {:>4.4f} Training Accuracy: {:.6f}'.format(i + 1, epoch_loss, epoch_acc))
valid_image, valid_label = next(val_batch_gen)
valid_this_loss, valid_this_acc = sess.run([cost, accuracy], feed_dict = {
input_img: valid_image,
y_true: valid_label,
lr_in: lr,
keep_prob: 1.
})
valid_loss.append(valid_this_loss)
valid_acc.append(valid_this_acc)
print('Epoch {:>2} Loss: {:>4.4f} Validation Accuracy: {:.6f}'.format(i + 1,valid_this_loss, valid_this_acc))
# early stop
if valid_this_loss > np.min(valid_loss):
es_n += 1
lr_n += 1
else:
es_n = 0
lr_n = 0
saver.save(sess, os.path.join(os.getcwd(), 'bestsession.ckpt'))
# early stop
if es_n >= es_patience:
print("-----------early stopping-------------")
break
# adaptive learning rate
if lr_n >= lr_patience:
lr *= lr_decay_rate
lr_n = 0
print("-----------adjust learning rate------------")
# Save Model
save_path = saver.save(sess, save_model_path)
print('-----model save ------')
----------- 18/09/07------------
I can always reproduce the same result.
And this is my code, I wrote it in Jupyter. But sorry I can't upload the training data:
https://drive.google.com/open?id=1uUE32KrNmWnhLbV8z-fyHSMu6zGCCG_e

I use CNN to classification MNIST with TensorFlow. But I want to read the dataset with TFRecordrs. However I only get 10% accuracy

I want to classification the dataset of mnist in my way with TensorFlow.
First, converting the dataset into a TFRecords file.
Then, reading this file with tf.TFRecodsReader.
Finally,training the dataset.
Existing problems：The code doesn't have any syntax errors. But，I only get 10% accuracy on the test data.
train.py:
import tensorflow as tf
import Net
import os
import numpy as np
import datetime
import time
import tfrecords as rd
BATCH_SIZE = 100
LEARNING_RATE_BASE = 0.01
LEARNING_RATE_DECAY = 0.99
REGULARIZATION_RATE = 0.0001
TRAINING_STEPS = 10000
MOVING_AVERAGE_DECAY = 0.99
MODEL_SAVE_PATH = "Model/"
MODEL_NAME = "model"
def train():
x = tf.placeholder(tf.float32, [
BATCH_SIZE,
Net.IMAGE_SIZE,
Net.IMAGE_SIZE,
Net.NUM_CHANNELS],
name='x-input')
y_ = tf.placeholder(tf.float32, [None, Net.OUTPUT_NODE], name='y-input')
regularizer = tf.contrib.layers.l2_regularizer(REGULARIZATION_RATE)
y = Net.inference(x, True, regularizer)
global_step = tf.Variable(0, trainable=False)
variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
variables_averages_op = variable_averages.apply(tf.trainable_variables())
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))
cross_entropy_mean = tf.reduce_mean(cross_entropy)
loss = cross_entropy_mean + tf.add_n(tf.get_collection('losses'))
learning_rate = tf.train.exponential_decay(
LEARNING_RATE_BASE,
global_step,
55000 / BATCH_SIZE, LEARNING_RATE_DECAY,
staircase=True)
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)
with tf.control_dependencies([train_step, variables_averages_op]):
train_op = tf.no_op(name='train')
init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
img, one_hot = rd.read_and_decode("./tfrecords/train.tfrecords")
img_batch, label_batch = tf.train.shuffle_batch(tensors = [img, one_hot], batch_size = BATCH_SIZE, capacity = 10000 + 3 * 100, min_after_dequeue = 10000)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(init_op)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord = coord, sess = sess)
try:
steps = 1
while not coord.should_stop():
if steps > TRAINING_STEPS:
break
xs, ys = sess.run([img_batch, label_batch])
_, loss_value, step = sess.run([train_op, loss, global_step], feed_dict={x: xs, y_: ys})
if steps % 1000 == 0:
print("After %d training step(s), loss on training batch is %g." % (step, loss_value))
saver.save(sess, os.path.join(MODEL_SAVE_PATH, MODEL_NAME), global_step=global_step)
steps += 1
except tf.errors.OutOfRangeError:
print("Done training after reading all data")
finally:
coord.request_stop()
coord.join(threads)
sess.close()
def main(argv=None):
start_time = datetime.datetime.now()
print("start_time = "),
print(start_time)
rd.create_record("train")
train_start_time = datetime.datetime.now()
print("train_start_time = " ),
print( train_start_time)
train()
end_time = datetime.datetime.now()
print("end_time = " ),
print(end_time)
if __name__ == '__main__':
main()
Net.py:
import tensorflow as tf
INPUT_NODE = 784
OUTPUT_NODE = 10
IMAGE_SIZE = 28
NUM_CHANNELS = 1
NUM_LABELS = 10
CONV1_DEEP = 32
CONV1_SIZE = 5
CONV2_DEEP = 64
CONV2_SIZE = 5
FC_SIZE = 512
def inference(input_tensor, train, regularizer):
with tf.variable_scope('layer1-conv1'):
conv1_weights = tf.get_variable(
"weight", [CONV1_SIZE, CONV1_SIZE, NUM_CHANNELS, CONV1_DEEP],
initializer=tf.truncated_normal_initializer(stddev=0.1))
conv1_biases = tf.get_variable("bias", [CONV1_DEEP], initializer=tf.constant_initializer(0.0))
conv1 = tf.nn.conv2d(input_tensor, conv1_weights, strides=[1, 1, 1, 1], padding='SAME')
relu1 = tf.nn.relu(tf.nn.bias_add(conv1, conv1_biases))
with tf.name_scope("layer2-pool1"):
pool1 = tf.nn.max_pool(relu1, ksize = [1,2,2,1],strides=[1,2,2,1],padding="SAME")
with tf.variable_scope("layer3-conv2"):
conv2_weights = tf.get_variable(
"weight", [CONV2_SIZE, CONV2_SIZE, CONV1_DEEP, CONV2_DEEP],
initializer=tf.truncated_normal_initializer(stddev=0.1))
conv2_biases = tf.get_variable("bias", [CONV2_DEEP], initializer=tf.constant_initializer(0.0))
conv2 = tf.nn.conv2d(pool1, conv2_weights, strides=[1, 1, 1, 1], padding='SAME')
relu2 = tf.nn.relu(tf.nn.bias_add(conv2, conv2_biases))
with tf.name_scope("layer4-pool2"):
pool2 = tf.nn.max_pool(relu2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
pool_shape = pool2.get_shape().as_list()
nodes = pool_shape[1] * pool_shape[2] * pool_shape[3]
reshaped = tf.reshape(pool2, [pool_shape[0], nodes])
with tf.variable_scope('layer5-fc1'):
fc1_weights = tf.get_variable("weight", [nodes, FC_SIZE],
initializer=tf.truncated_normal_initializer(stddev=0.1))
if regularizer != None: tf.add_to_collection('losses', regularizer(fc1_weights))
fc1_biases = tf.get_variable("bias", [FC_SIZE], initializer=tf.constant_initializer(0.1))
fc1 = tf.nn.relu(tf.matmul(reshaped, fc1_weights) + fc1_biases)
if train: fc1 = tf.nn.dropout(fc1, 0.5)
with tf.variable_scope('layer6-fc2'):
fc2_weights = tf.get_variable("weight", [FC_SIZE, NUM_LABELS],
initializer=tf.truncated_normal_initializer(stddev=0.1))
if regularizer != None: tf.add_to_collection('losses', regularizer(fc2_weights))
fc2_biases = tf.get_variable("bias", [NUM_LABELS], initializer=tf.constant_initializer(0.1))
logit = tf.matmul(fc1, fc2_weights) + fc2_biases
return logit
tfrecords.py:
import os
import tensorflow as tf
from PIL import Image
import Net
def create_record(op_type):
writer = tf.python_io.TFRecordWriter("./tfrecords/" + op_type + ".tfrecords")
f = open("./" + op_type + ".txt", 'r')
img_num = 0
for line in f.readlines():
img_num += 1
if img_num % 2000 == 0:
print("already read in %d images." % (img_num))
str_split = line.split()
img_path = "./" + str_split[0]
index = int(str_split[1])
img = Image.open(img_path)
img = img.resize((Net.IMAGE_SIZE, Net.IMAGE_SIZE))
img_raw = img.tobytes()
example = tf.train.Example(features=tf.train.Features(feature={
'label': tf.train.Feature(int64_list=tf.train.Int64List(value=[index])),
'img_raw': tf.train.Feature(bytes_list=tf.train.BytesList(value=[img_raw]))
}))
writer.write(example.SerializeToString())
writer.close()
def read_and_decode(filename):
filename_queue = tf.train.string_input_producer([filename], shuffle = True)
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(serialized_example,
features={
'label': tf.FixedLenFeature([], tf.int64),
'img_raw' : tf.FixedLenFeature([], tf.string),
})
img = tf.decode_raw(features['img_raw'], tf.uint8)
img = tf.reshape(img, [Net.IMAGE_SIZE, Net.IMAGE_SIZE, Net.NUM_CHANNELS])
img = 1 - tf.cast(img, tf.float32) * (1. / 255)
label = tf.cast(features['label'], tf.int32)
one_hot = tf.one_hot(label, 10, dtype = tf.float32)
one_hot = tf.reshape(one_hot, [-1])
return img, one_hot
Eval.py:
import time
import math
import tensorflow as tf
import numpy as np
import Net
import Train
import tfrecords as rd
def evaluate():
with tf.Graph().as_default() as g:
x = tf.placeholder(tf.float32, [
10000,
Net.IMAGE_SIZE,
Net.IMAGE_SIZE,
Net.NUM_CHANNELS],
name='x-input')
y_ = tf.placeholder(tf.float32, [None, Net.OUTPUT_NODE], name='y-input')
#validate_feed = {x: mnist.test.images, y_: mnist.test.labels}
global_step = tf.Variable(0, trainable=False)
regularizer = tf.contrib.layers.l2_regularizer(Train.REGULARIZATION_RATE)
y = Net.inference(x, False, regularizer)
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
variable_averages = tf.train.ExponentialMovingAverage(Train.MOVING_AVERAGE_DECAY)
variables_to_restore = variable_averages.variables_to_restore()
saver = tf.train.Saver(variables_to_restore)
for i in range(1):
img, one_hot = rd.read_and_decode("./tfrecords/test.tfrecords")
img_batch, label_batch = tf.train.shuffle_batch(tensors = [img, one_hot], batch_size = 10000, capacity = 10000 + 3 * 100, min_after_dequeue = 10000)
with tf.Session() as sess:
ckpt = tf.train.get_checkpoint_state(Train.MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
#the reason of this error!!!
#tf.global_variables_initializer().run()
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord = coord, sess = sess)
try:
xs, ys = sess.run([img_batch, label_batch])
accuracy_score = sess.run(accuracy, feed_dict={x:xs, y_:ys})
print("After %s training step(s), test accuracy = %g" % (global_step, accuracy_score))
except tf.errors.OutOfRangeError:
print("Done testting after reading all data")
finally:
coord.request_stop()
coord.join(threads)
sess.close()
else:
print('No checkpoint file found')
return
def main(argv=None):
rd.create_record("test")
evaluate()
if __name__ == '__main__':
main()
Now, I have solved this problem.
My code and other flie: https://github.com/xmy7216/MNIST_classification.git
Linux: Red Hat Enterprise Linux Server release 7.2 (Maipo)
GPU: Tesla P4
TensorFlow: 1.3
Python:2.7.5

I find the reason. Because after I load the model using saver.restore, I initialize the variables again by tf.global_variables_initializer().run().
How stupid I am!

Imported Library considered as a variable in python

When I run this code, I get the error:
"UnboundLocalError: local variable 'tf' referenced before assignment"
In the Line where I Declare the weights . Why is tf being considered as a variable which is not being assigned ?
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/tmp/data/",one_hot = True)
n_nodes_hl1 = 500
n_nodes_hl2 = 500
n_nodes_hl3 = 500
n_classes = 10
batch_size = 128
x = tf.placeholder('float',[None,784]) #THrows error if the matrix is not 784-dim vec
y = tf.placeholder('float')
def convolutional_neural_network_model(x):
#The Line with the Error
weights = {'W_conv1':tf.Variable(tf.random_normal([5,5,1,32])),
'W_conv2':tf.Variable(tf.random_normal([5,5,32,64])),
'W_fc':tf.Variable(tf.random_normal([7*7*64,1024])),
'out':tf.Variable(tf.random_normal([1024, n_classes]))}
biases = {'b_conv1':tf.Variable(tf.random_normal([32])),
'b_conv2':tf.Variable(tf.random_normal([64])),
'b_fc':tf.Variable(tf.random_normal([1024])),
'out':tf.Variable(tf.random_normal([n_classes]))}
x = tf.reshape(x,shape=[-1,28,28,1])
conv1 = conv2d(x,weights['W_conv1'])
conv1 = maxpool2d(conv1)
conv2 = conv2d(conv1,weights['W_conv2'])
conv2 = maxpool2d(conv2)
fc = tf.reshape(conv2,[-1,7*7*64])
tf = tf.nn.relu(tf.matmul(fc,weights['W_fc']) + biases['b_fc'])
output = tf.matmul(fc,weights['out']) + biases['out']
return output
def train_neural_network(x):
prediction = convolutional_neural_network_model(x)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = prediction,labels = y))
# learning_rate = 0.001
optimizer = tf.train.AdamOptimizer().minimize(cost)
#
hm_epochs = 10
#
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
#
for epoch in range(hm_epochs):
epoch_loss = 0
for _ in range(int(mnist.train.num_examples/batch_size)):
epoch_x,epoch_y = mnist.train.next_batch(batch_size)
_, c = sess.run([optimizer,cost], feed_dict={x: epoch_x, y:epoch_y})
epoch_loss += c
print('Epoch',epoch,'/',hm_epochs,' loss :',epoch_loss)
#
correct = tf.equal(tf.argmax(prediction,1),tf.argmax(y,1))
#
accuracy = tf.reduce_mean(tf.cast(correct,'float'))
print('Accuracy :',accuracy.eval({x:mnist.test.images,y:mnist.test.labels}))
#
def conv2d(x,W):
return tf.nn.conv2d(x,W,strides=[1,1,1,1],padding='SAME')
def maxpool2d(x):
return tf.nn.max_pool(x,ksize = [1,2,2,1],strides=[1,2,2,1],padding='SAME')
if __name__ == '__main__':
print('Begin')
train_neural_network(x)
print('Done')

After importing tensorflow as tf, you redefine tf in the following line:
tf = tf.nn.relu(tf.matmul(fc,weights['W_fc']) + biases['b_fc'])
Changing the name of this variable will fix the problem.

tf = tf.nn.relu(tf.matmul(fc,weights['W_fc']) + biases['b_fc'])
You shouldn't use tf as the name of a variable if you import tensorflow as tf; it is unclear whether you are referring to the variable or the module.