We Keep Coding

Why does Chainer Batchnormalization not work well with mnist_nn

Chainer batchnormalization does not work well with my code although batch normalization of tensorflow works. I use the dataset mnist the code below shows.
Using chainer(version=6.1.0), without batchnormalization, the validation accuracy is within 0.97 and 0.98 after 100 epochs whereas with batchnormalization, it is less than 0.80 after 100 epochs.
When I use the same way with tensorflow(version=1.14.0), the validation accuracy is around 0.98 in a both way, with batchnormalization or without batchnormalization.
This is the part of my code.
The number of epoch is 100, and its batchsize is 1000. I use the Adam as the optimizer with learning_rate 0.01.
dataset, train data, validation data
(train_images, train_labels), (test_images, test_labels) = mnist.load_data()
train_images=train_images.reshape(60000, 28*28)
test_images = test_images.reshape(10000, 28*28)
x_train = train_images.astype('float32')/255
y_train = train_labels.astype('int32')
x_val = test_images.astype('float32')/255
y_val = test_labels.astype('int32')
model and condition (chainer)
# Define model
class MyModel(Chain):
def __init__(self,n_in=784,n_hidden=100,n_out=10):
initializer = chainer.initializers.HeNormal()
super().__init__()
with self.init_scope():
self.l1=L.Linear(n_in, n_hidden, initialW=initializer)
self.l2=L.Linear(n_hidden, n_hidden, initialW=initializer)
self.l3=L.Linear(n_hidden, n_out, initialW=initializer)
self.bn=L.BatchNormalization(n_hidden, decay=0.99, eps=0.001)
def forward(self,x):
h = F.relu(self.bn(self.l1(x)))
h = F.relu(self.bn(self.l2(h)))
return self.l3(h)
model = MyModel()
optimizer = optimizers.Adam()
optimizer.setup(model)
n_epoch = 100
n_batchsize = 1000
model and condition (tensorflow)
n_inputs = 28 * 28
n_hidden1 = 100
n_hidden2 = 100
n_outputs = 10
batch_norm_momentum = 0.9
X = tf.placeholder(tf.float32, shape=(None, n_inputs), name="X")
y = tf.placeholder(tf.int32, shape=(None), name="y")
training = tf.placeholder_with_default(False, shape=(), name='training')
with tf.name_scope("dnn"):
he_init = tf.variance_scaling_initializer()
my_batch_norm_layer = partial(tf.layers.batch_normalization,
training=training,
momentum=batch_norm_momentum)
my_dense_layer = partial(tf.layers.dense,
kernel_initializer=he_init)
hidden1 = my_dense_layer(X, n_hidden1, name="hidden1")
bn1 = tf.nn.relu(my_batch_norm_layer(hidden1))
hidden2 = my_dense_layer(bn1, n_hidden2, name="hidden2")
bn2 = tf.nn.relu(my_batch_norm_layer(hidden2))
logits_before_bn = my_dense_layer(bn2, n_outputs, name="outputs")
logits = my_batch_norm_layer(logits_before_bn)
with tf.name_scope("loss"):
xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)
loss = tf.reduce_mean(xentropy, name="loss")
with tf.name_scope("train"):
optimizer = tf.train.AdamOptimizer(learning_rate=0.001)
training_op = optimizer.minimize(loss)
with tf.name_scope("eval"):
correct = tf.nn.in_top_k(logits, y, 1)
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
All code I use with chainer
import numpy as np
import chainer
from chainer import cuda, Function, gradient_check, report, training, utils, Variable
from chainer import datasets, iterators, optimizers, serializers
from chainer import Link, Chain, ChainList
import chainer.functions as F
import chainer.links as L
from chainer.training import extensions
from keras.datasets import mnist
import cupy as cp
(train_images, train_labels), (test_images, test_labels) = mnist.load_data()
train_images=train_images.reshape(60000, 28*28)
test_images = test_images.reshape(10000, 28*28)
x_train = train_images.astype('float32')/255
y_train = train_labels.astype('int32')
x_val = test_images.astype('float32')/255
y_val = test_labels.astype('int32')
# Define model
class MyModel(Chain):
def __init__(self,n_in=784,n_hidden=100,n_out=10):
initializer = chainer.initializers.HeNormal()
super().__init__()
with self.init_scope():
self.l1=L.Linear(n_in, n_hidden, initialW=initializer)
self.l2=L.Linear(n_hidden, n_hidden, initialW=initializer)
self.l3=L.Linear(n_hidden, n_out, initialW=initializer)
self.bn=L.BatchNormalization(n_hidden, decay=0.9, eps=0.001)
def forward(self,x):
h = F.relu(self.bn(self.l1(x)))
h = F.relu(self.bn(self.l2(h)))
return self.l3(h)
# define optimizer
model = MyModel()
optimizer = optimizers.Adam(alpha=0.01)
optimizer.setup(model)
## learn network
n_epoch = 100
n_batchsize = 1000
iteration = 0
gpu_id = 0
cuda.get_device(gpu_id).use()
# send the network to gpu memory
model.to_gpu(gpu_id)
print("epoch train/loss val/loss train/acc val/acc")
for epoch in range(n_epoch):
# order dataset randomly
order = np.random.permutation(range(len(x_train)))
loss_list = []
accuracy_list = []
for i in range(0, len(order), n_batchsize):
index = order[i:i+n_batchsize]
x_train_batch = x_train[index,:]
y_train_batch = y_train[index]
x_train_batch = cp.asarray(x_train_batch)
y_train_batch = cp.asarray(y_train_batch)
output_train = model(x_train_batch)
loss_train_batch = F.softmax_cross_entropy(output_train, y_train_batch)
accuracy_train_batch = F.accuracy(output_train, y_train_batch)
loss_list.append(cuda.to_cpu(loss_train_batch.array))
accuracy_list.append(cuda.to_cpu(accuracy_train_batch.array))
model.cleargrads()
loss_train_batch.backward()
optimizer.update()
iteration += 1
loss_train = np.mean(loss_list)
accuracy_train = np.mean(accuracy_list)
# after one epoch, evaluate with validation data
x_val = cp.asarray(x_val)
y_val = cp.asarray(y_val)
with chainer.using_config('train', False), chainer.using_config('enable_backprop', False):
output_val = model(x_val)
loss_val = F.softmax_cross_entropy(output_val, y_val)
loss_val = cuda.to_cpu(loss_val.array)
accuracy_val = F.accuracy(output_val, y_val)
accuracy_val = cuda.to_cpu(accuracy_val.array)
print('{0:>4d} {1:>10.4f} {2:>10.4f} {3:>10.4f} {4:>10.4f}'.format(epoch,loss_train,loss_val,accuracy_train,accuracy_val))
All code I use with tensorflow
python
import tensorflow as tf
from keras.datasets import mnist
from functools import partial
import numpy as np
def shuffle_batch(X, y, batch_size):
rnd_idx = np.random.permutation(len(X))
n_batches = len(X) // batch_size
for batch_idx in np.array_split(rnd_idx, n_batches):
X_batch, y_batch = X[batch_idx], y[batch_idx]
yield X_batch, y_batch
(train_images, train_labels), (test_images, test_labels) = mnist.load_data()
train_images=train_images.reshape(60000, 28*28)
test_images = test_images.reshape(10000, 28*28)
X_train = train_images.astype('float32')/255
y_train = train_labels.astype('int32')
X_valid = test_images.astype('float32')/255
y_valid = test_labels.astype('int32')
n_inputs = 28 * 28
n_hidden1 = 100
n_hidden2 = 100
n_outputs = 10
batch_norm_momentum = 0.9
X = tf.placeholder(tf.float32, shape=(None, n_inputs), name="X")
y = tf.placeholder(tf.int32, shape=(None), name="y")
training = tf.placeholder_with_default(False, shape=(), name='training')
with tf.name_scope("dnn"):
he_init = tf.variance_scaling_initializer()
my_batch_norm_layer = partial(tf.layers.batch_normalization,
training=training,
momentum=batch_norm_momentum)
my_dense_layer = partial(tf.layers.dense)
hidden1 = my_dense_layer(X, n_hidden1, name="hidden1")
bn1 = tf.nn.relu(my_batch_norm_layer(hidden1))
hidden2 = my_dense_layer(bn1, n_hidden2, name="hidden2")
bn2 = tf.nn.relu(my_batch_norm_layer(hidden2))
logits_before_bn = my_dense_layer(bn2, n_outputs, name="outputs")
logits = my_batch_norm_layer(logits_before_bn)
with tf.name_scope("loss"):
xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)
loss = tf.reduce_mean(xentropy, name="loss")
learning_rate = 0.01
with tf.name_scope("train"):
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
training_op = optimizer.minimize(loss)
with tf.name_scope("eval"):
correct = tf.nn.in_top_k(logits, y, 1)
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
init = tf.global_variables_initializer()
saver = tf.train.Saver()
n_epochs = 100
batch_size = 1000
extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
print("epoch train/loss val/loss train/acc val/acc")
with tf.Session() as sess:
init.run()
for epoch in range(n_epochs):
loss_list = []
accuracy_list = []
for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):
sess.run([training_op, extra_update_ops],
feed_dict={training: True, X: X_batch, y: y_batch})
loss_batch = loss.eval(feed_dict={X: X_batch, y: y_batch})
accuracy_batch = accuracy.eval(feed_dict={X: X_batch, y: y_batch})
loss_list.append(loss_batch)
accuracy_list.append(accuracy_batch)
loss_val = loss.eval(feed_dict={X: X_valid, y: y_valid})
accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})
print('{0:>4d} {1:>10.4f} {2:>10.4f} {3:>10.4f} {4:>10.4f}'
.format(epoch,np.mean(loss_list),loss_val,np.mean(accuracy_list),accuracy_val))
I expect batch normalization with chainer would be around 98% but it got less than 80%.
Am I using the batchnormalization with chainer in a wrong way, or the structure of batchnormalization differs a lot between chainer and tensorflow??

In order to use different batch statistics between the layers, the model definition has to be like the following code, which achieves 98% validation accuracy after 100 epochs in my environment.
class MyModel(Chain):
def __init__(self,n_in=784,n_hidden=100,n_out=10):
initializer = chainer.initializers.HeNormal()
super().__init__()
with self.init_scope():
self.l1=L.Linear(n_in, n_hidden, initialW=initializer)
self.l2=L.Linear(n_hidden, n_hidden, initialW=initializer)
self.l3=L.Linear(n_hidden, n_out, initialW=initializer)
self.bn1=L.BatchNormalization(n_hidden, decay=0.9, eps=0.001)
self.bn2=L.BatchNormalization(n_hidden, decay=0.9, eps=0.001)
def forward(self,x):
h = F.relu(self.bn1(self.l1(x)))
h = F.relu(self.bn2(self.l2(h)))
return self.l3(h)

Tensorflow save/load frozen tf.graph and run classification on loaded graph

as I have mentioned in the topic I would like to save my tf.graph into a frozen_graph.pb file. This should save space later I will try to run it an a jetson tx2. I have made a short MNIST example describing my problem. I run tf 1.7 on python 3.5.
Question1: As far as I understood my freeze_graph method takes a checkpoint file transfers all variables to constants except the ones i define with the second parameter. When I try to get the correct tensorname I wrote loggits.name but I get an error no Tensor with that name found in graph.
Question2: After that I would be able to extract a frozen graph, how can i load it back and run an classification on that.
My Code is attached and should work in a single py file.
Thank you very much in advance
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
from tensorflow.examples.tutorials.mnist import input_data
import os
import time
import tensorflow as tf
import os
import argparse
#METHODS I WANT TO TEST
#TAKE THE CHECKPOINT FILE AND DELETE ALL NOTES THAT ARE NOT USEFUL
def freeze_graph(checkpoint_directory,output_node_names):
#checkpoint = tf.train.get_checkpoint_state(checkpoint_directory)
print(checkpoint_directory)
checkpoint = tf.train.get_checkpoint_state(checkpoint_directory)
input_checkpoint = checkpoint.model_checkpoint_path
absolute_model_dir = str(os.sep).join(input_checkpoint.split(os.sep)[:-1])
output_graph = absolute_model_dir + "/frozen_model.pb"
clear_devices = True
with tf.Session(graph = tf.Graph()) as sess:
#import the metagraph in default graph
saver = tf.train.import_meta_graph(input_checkpoint + '.meta',clear_devices=clear_devices)
#restore the weights
saver.restore(sess,input_checkpoint)
#wrap variables to constants
[print(n.name) for n in tf.get_default_graph().as_graph_def().node]
output_graph_def = tf.graph_util.convert_variables_to_constants(sess, tf.get_default_graph().as_graph_def(),output_node_names.split(","))
with tf.gfile.GFile(output_graph, "wb") as f:
f.write(output_graph_def.SerializeToString())
print("%d ops in the final graph." %len(output_graph_def.node))
return output_graph_def
#HERE IS THE METHOD THAT ALLOWS ME TO LOAD MY FROZEN GRAPH AS GRAPH
def load_graph(frozen_graph_filename):
with tf.gfile.GFile(frozen_graph_filename,"rb") as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
with tf.Graph().as_default() as graph:
tf.import_graph_def(graph_def, name = "prefix")
return graph
#get the data
mnist = input_data.read_data_sets("MNIST_data/",one_hot=True)
#NETWORK PARAMETERS
learning_rate = 0.01
dropout = 0.75
display_step = 1
filter_height = 5
filter_width = 5
depth_in = 1
depth_out1 = 64
depth_out2 = 128
#PARAMETERS OF THE DATASET
input_height = 28
input_width = 28
n_classes = 10
#TRAINING PARAMETERS
epochs = 1
batch_size = 256
num_batches = int(mnist.train.num_examples/batch_size)
x = tf.placeholder(tf.float32,[None,28*28],name = "input")
y = tf.placeholder(tf.float32,[None,n_classes])
keep_prob = tf.placeholder(tf.float32)
weights = {'wc1': tf.Variable(tf.random_normal([filter_height,filter_width,depth_in,depth_out1])),
'wc2': tf.Variable(tf.random_normal([filter_height, filter_width, depth_out1, depth_out2])),
'wd1': tf.Variable(tf.random_normal([int(input_height/4)*int(input_height/4)*depth_out2,1024])),
'out': tf.Variable(tf.random_normal([1024,n_classes]))}
biases = {'bc1': tf.Variable(tf.random_normal([depth_out1])),
'bc2': tf.Variable(tf.random_normal([depth_out2])),
'bd1': tf.Variable(tf.random_normal([1024])),
'out': tf.Variable(tf.random_normal([n_classes]))}
#DEFINE YOUR NEURAL NETWORKS LAYER OPERATIONS
def ops_conv2d(x,W,b,strides = 1, add_bias = True, activation = tf.nn.relu, use_activation = True):
x = tf.nn.conv2d(x,W,strides = [1,strides,strides,1],padding = 'SAME')
x = tf.nn.bias_add(x,b)
if use_activation:
return activation(x)
else:
return x
def ops_maxpool2d(x,stride=2):
return tf.nn.max_pool(x,ksize=[1,stride,stride,1],strides = [1,stride,stride,1], padding = 'SAME' )
def ops_dropout(input_fully_connected,dropout):
return tf.nn.dropout(input_fully_connected,dropout)
def ops_fullyconnected(input, activation = tf.nn.relu, use_activation = True):
fc = tf.reshape(input,[-1,weights['wd1'].get_shape().as_list()[0]])
fc = tf.add(tf.matmul(fc,weights['wd1']),biases['bd1'])
if use_activation:
return activation(fc)
else:
return fc
#DEFINE NETWORK ARCHTEKTURE (FORWARDPASS)
def build_network(x,weights,biases,dropout):
x = tf.reshape(x,shape=(-1,28,28,1))
conv_layer_1 = ops_conv2d(x,weights['wc1'],biases['bc1'],activation=tf.nn.relu, use_activation=True)
conv_layer_1 = ops_maxpool2d(conv_layer_1,2)
conv_layer_2 = ops_conv2d(conv_layer_1,weights['wc2'],biases['bc2'],activation=tf.nn.relu, use_activation=True)
conv_layer_2 = ops_maxpool2d(conv_layer_2,2)
fc1 = ops_fullyconnected(conv_layer_2, activation=tf.nn.relu, use_activation=True)
fc1 = ops_dropout(fc1,dropout)
logits = tf.add(tf.matmul(fc1,weights['out']),biases['out'],name = "logits")
return logits
#DEFINE TENSORFLOW BACKPROPAGATION OBJECTS (BACKWARDPASS)
logits = build_network(x,weights,biases,keep_prob)
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = logits,labels = y))
#CHOSE AN OPTIMIZER
optimizer = tf.train.AdadeltaOptimizer(learning_rate=learning_rate).minimize(loss=loss)
predicted_labels = tf.equal(tf.argmax(logits,1),tf.argmax(y,1))
#EVALUATION PARAMETERS
acc = tf.reduce_mean(tf.cast(predicted_labels,tf.float32))
#NOW INITIALIZE ALL TF VARIABLES
init = tf.global_variables_initializer()
saver = tf.train.Saver(max_to_keep=10)
#NOW START THE SESSION AND EXECUTE THE GRAPH
with tf.Session() as sess:
sess.run(init)
for i in range(epochs):
save_path = saver.save(sess, os.curdir + "checkpoints/MNIST_TEST.ckpt")
for j in range(num_batches):
batch_x, batch_y = mnist.train.next_batch(batch_size)
sess.run(optimizer, feed_dict={x:batch_x,y:batch_y,keep_prob:dropout})
losses,accs = sess.run([loss,acc],feed_dict={x:batch_x,y:batch_y,keep_prob:1.})
if epochs % display_step == 0:
print("EPOCH:",'%04d' % (i+1),
"loss =", "{:.9f}".format(losses),
"acc =", "{:.5f}".format(accs))
print("TRAINING COMPLETED")
#START PREDICTIONS
predicted_label = sess.run(logits,feed_dict={x:mnist.test.images[:256],keep_prob:1.})
test_classes = np.argmax(predicted_label,1)
print("TEST ACCURACY:",sess.run(acc,feed_dict={x:mnist.test.images[:256], y:mnist.test.labels[:256],keep_prob:1.}))
f,a = plt.subplots(1,10,figsize = (10,2))
for i in range(10):
a[i].imshow(np.reshape(mnist.test.images[i],(28,28)))
print( test_classes[i])
print("TOTAL EXAMPLE FINNISHED")
freeze_graph(os.curdir + "checkpoints" + os.sep, logits.name)
graph = load_graph(os.curdir + os.sep + "checkpoints" + os.sep + "frozen_model.pb")
with tf.Session(graph) as sess:
sess.run(init)
predicted_label = sess.run(logits, feed_dict={x: mnist.test.images[:256], keep_prob: 1.})
print(predicted_label)

if anybody has the same problem here is a description how i solved it.
Saving and Loading the data:
First of all note that I have now a different pipeline. First of all I save the session in a saver (ckpt files). Afterwards I construct a metagaph (graph.pb). This graph is then transfered into a frozen graph (frozen.pb). To load the frozen graph I use the load_frozen_graph_from_session method. Inside that method I also test a forward pass through my network.
Running an inference on the loaded graph:
First I name my tensors x (name = "input") this will result in a tensorname ("input:0")
so when you try to fill this placeholder in the new session you need predicted_label = sess.run("output:0", feed_dict={"input:0":mnist.test.images[:256], "keep_prob:0": 1.})
The output is the logit and not the prediction inside my network. This is because if you run the session it will run until it hits the variable you want to fetch. would I take the prediction I need also the placeholder for my y (name=label).
Here is the full code:
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
from tensorflow.examples.tutorials.mnist import input_data
import os
import time
import tensorflow as tf
import os
import argparse
from tensorflow.python.platform import gfile
from tensorflow.python.framework.graph_util import convert_variables_to_constants
#METHODS I WANT TO TEST
def freeze_graph_from_Session(sess,saver):
# convert_variables_to_constants(sess, input_graph_def, output_node_names, variable_names_whitelist=None)
save_graph(sess,saver)
with gfile.FastGFile("./tmp/" + "graph.pb", 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
frozen_graph_def = convert_variables_to_constants(sess, graph_def, ["output"])
with tf.gfile.GFile("./tmp/" + "frozen.pb", "wb") as f:
f.write(frozen_graph_def.SerializeToString())
def save_graph(sess, saver):
saver.save(sess, "./tmp/model", write_meta_graph=True, global_step=1)
with open("./tmp/" + "graph.pb", 'wb') as f:
f.write(sess.graph_def.SerializeToString())
#sess.close()
def load_frozen_graph_from_session():
filename = "./tmp/" + "frozen.pb"
print("LOADING GRAPH")
with tf.gfile.GFile(filename, "rb") as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
print("OPEN GRAPH")
with tf.Graph().as_default() as graph:
print("DEFINE INPUT")
new_input = tf.placeholder(tf.float32, [None, 28 * 28], name="new_input")
print("DEFINE INPUT MAP")
tf.import_graph_def(
graph_def,
# usually, during training you use queues, but at inference time use placeholders
# this turns into "input
input_map={"input:0": new_input},
return_elements=None,
# if input_map is not None, needs a name
name="bla",
op_dict=None,
producer_op_list=None
)
checkpoint_path = tf.train.latest_checkpoint("./tmp/")
with tf.Session(graph=graph) as sess:
saver = tf.train.import_meta_graph(checkpoint_path + ".meta", import_scope=None)
saver.restore(sess, checkpoint_path)
print("TRY FORWARD RUN THROUGH LOADED GRAPH")
predicted_label = sess.run("output:0", feed_dict={"input:0":mnist.test.images[:256], "keep_prob:0": 1.})
print("output", predicted_label)
f, a = plt.subplots(1, 10, figsize=(10, 2))
test_classes = np.argmax(predicted_label, 1)
for i in range(10):
a[i].imshow(np.reshape(mnist.test.images[i], (28, 28)))
print(test_classes[i])
print ("output:", test_classes)
#TAKE THE CHECKPOINT FILE AND DELETE ALL NOTES THAT ARE NOT USEFUL
def freeze_graph(checkpoint_directory,output_node_names):
#checkpoint = tf.train.get_checkpoint_state(checkpoint_directory)
print(checkpoint_directory)
checkpoint = tf.train.get_checkpoint_state(checkpoint_directory)
input_checkpoint = checkpoint.model_checkpoint_path
absolute_model_dir = str(os.sep).join(input_checkpoint.split(os.sep)[:-1])
output_graph = absolute_model_dir + "/frozen_model.pb"
clear_devices = True
with tf.Session(graph = tf.Graph()) as sess:
#import the metagraph in default graph
saver = tf.train.import_meta_graph(input_checkpoint + '.meta',clear_devices=clear_devices)
#restore the weights
saver.restore(sess,input_checkpoint)
#wrap variables to constants
[print(n.name) for n in tf.get_default_graph().as_graph_def().node]
output_graph_def = tf.graph_util.convert_variables_to_constants(sess, tf.get_default_graph().as_graph_def(),output_node_names.split(","))
with tf.gfile.GFile(output_graph, "wb") as f:
f.write(output_graph_def.SerializeToString())
print("%d ops in the final graph." %len(output_graph_def.node))
return output_graph_def
#HERE IS THE METHOD THAT ALLOWS ME TO LOAD MY FROZEN GRAPH AS GRAPH
def load_graph(frozen_graph_filename):
with tf.gfile.GFile(frozen_graph_filename,"rb") as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
with tf.Graph().as_default() as graph:
tf.import_graph_def(graph_def, name = "prefix")
return graph
#get the data
mnist = input_data.read_data_sets("MNIST_data/",one_hot=True)
print(mnist.test.labels[:256])
print("load_freeze_graph_from_session: STARTED")
load_frozen_graph_from_session()
print("load_freeze_graph_from_session: ENDED")
exit()
#NETWORK PARAMETERS
learning_rate = 0.01
dropout = 0.75
display_step = 1
filter_height = 5
filter_width = 5
depth_in = 1
depth_out1 = 64
depth_out2 = 128
#PARAMETERS OF THE DATASET
input_height = 28
input_width = 28
n_classes = 10
#TRAINING PARAMETERS
epochs = 1
batch_size = 256
num_batches = int(mnist.train.num_examples/batch_size)
x = tf.placeholder(tf.float32,[None,28*28],name="input")
y = tf.placeholder(tf.float32,[None,n_classes],name = "label")
keep_prob = tf.placeholder(tf.float32,name = "keep_prob")
weights = {'wc1': tf.Variable(tf.random_normal([filter_height,filter_width,depth_in,depth_out1])),
'wc2': tf.Variable(tf.random_normal([filter_height, filter_width, depth_out1, depth_out2])),
'wd1': tf.Variable(tf.random_normal([int(input_height/4)*int(input_height/4)*depth_out2,1024])),
'out': tf.Variable(tf.random_normal([1024,n_classes]))}
biases = {'bc1': tf.Variable(tf.random_normal([depth_out1])),
'bc2': tf.Variable(tf.random_normal([depth_out2])),
'bd1': tf.Variable(tf.random_normal([1024])),
'out': tf.Variable(tf.random_normal([n_classes]))}
#DEFINE YOUR NEURAL NETWORKS LAYER OPERATIONS
def ops_conv2d(x,W,b,strides = 1, add_bias = True, activation = tf.nn.relu, use_activation = True):
x = tf.nn.conv2d(x,W,strides = [1,strides,strides,1],padding = 'SAME')
x = tf.nn.bias_add(x,b)
if use_activation:
return activation(x)
else:
return x
def ops_maxpool2d(x,stride=2):
return tf.nn.max_pool(x,ksize=[1,stride,stride,1],strides = [1,stride,stride,1], padding = 'SAME' )
def ops_dropout(input_fully_connected,dropout):
return tf.nn.dropout(input_fully_connected,dropout)
def ops_fullyconnected(input, activation = tf.nn.relu, use_activation = True):
fc = tf.reshape(input,[-1,weights['wd1'].get_shape().as_list()[0]])
fc = tf.add(tf.matmul(fc,weights['wd1']),biases['bd1'])
if use_activation:
return activation(fc)
else:
return fc
#DEFINE NETWORK ARCHTEKTURE (FORWARDPASS)
def build_network(x,weights,biases,dropout):
x = tf.reshape(x,shape=(-1,28,28,1))
conv_layer_1 = ops_conv2d(x,weights['wc1'],biases['bc1'],activation=tf.nn.relu, use_activation=True)
conv_layer_1 = ops_maxpool2d(conv_layer_1,2)
conv_layer_2 = ops_conv2d(conv_layer_1,weights['wc2'],biases['bc2'],activation=tf.nn.relu, use_activation=True)
conv_layer_2 = ops_maxpool2d(conv_layer_2,2)
fc1 = ops_fullyconnected(conv_layer_2, activation=tf.nn.relu, use_activation=True)
fc1 = ops_dropout(fc1,dropout)
logits = tf.add(tf.matmul(fc1,weights['out']),biases['out'],name = "output")
return logits
#DEFINE TENSORFLOW BACKPROPAGATION OBJECTS (BACKWARDPASS)
logits = build_network(x,weights,biases,keep_prob)
#freeze_graph(os.curdir + "checkpoints" + os.sep, logits.name)
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = logits,labels = y))
#CHOSE AN OPTIMIZER
optimizer = tf.train.AdadeltaOptimizer(learning_rate=learning_rate).minimize(loss=loss)
predicted_labels = tf.equal(tf.argmax(logits,1),tf.argmax(y,1))
#EVALUATION PARAMETERS
acc = tf.reduce_mean(tf.cast(predicted_labels,tf.float32))
#NOW INITIALIZE ALL TF VARIABLES
init = tf.global_variables_initializer()
saver = tf.train.Saver(max_to_keep=10)
#NOW START THE SESSION AND EXECUTE THE GRAPH
with tf.Session() as sess:
sess.run(init)
for i in range(epochs):
save_path = saver.save(sess, os.curdir + "checkpoints/MNIST_TEST.ckpt")
for j in range(num_batches):
batch_x, batch_y = mnist.train.next_batch(batch_size)
sess.run(optimizer, feed_dict={x:batch_x,y:batch_y,keep_prob:dropout})
losses,accs = sess.run([loss,acc],feed_dict={x:batch_x,y:batch_y,keep_prob:1.})
if epochs % display_step == 0:
print("EPOCH:",'%04d' % (i+1),
"loss =", "{:.9f}".format(losses),
"acc =", "{:.5f}".format(accs))
print("TRAINING COMPLETED")
#START PREDICTIONS
predicted_label = sess.run(logits,feed_dict={x:mnist.test.images[:256],keep_prob:1.})
test_classes = np.argmax(predicted_label,1)
print("TEST ACCURACY:",sess.run(acc,feed_dict={x:mnist.test.images[:256], y:mnist.test.labels[:256],keep_prob:1.}))
f,a = plt.subplots(1,10,figsize = (10,2))
for i in range(10):
a[i].imshow(np.reshape(mnist.test.images[i],(28,28)))
print( test_classes[i])
print("TOTAL EXAMPLE FINNISHED")
#freeze_graph(os.curdir + "checkpoints"+os.sep,logits)
print("freeze_graph_from_session: STARTED")
freeze_graph_from_Session(sess,saver)
print("freeze_graph_from_session: ENDED")
print("load_freeze_graph_from_session: STARTED")
load_frozen_graph_from_session()
print("load_freeze_graph_from_session: ENDED")
#with tf.Session() as sess:
#
# sess.run(init)
# graph = load_graph(os.curdir + os.sep + "checkpoints" + os.sep + "frozen_model.pb")
# predicted_label = sess.run(logits, feed_dict={x: mnist.test.images[:256], keep_prob: 1.})
# print(predicted_label)
Thanks goes out to my self. :)

Training CNN model by using activation function Selu

I am training my own model using Tensorflow. However, I got some trouble when I change my activation function from Relu to Selu.
This is what happened. Learning curve drops accidentally and I have no idea about what's going on.
my learning curve
like this.
For what I have known, Selu can prevent overfitting, so I try to implement it in my model. Is there any tips, or any condition when I want to use Selu?
This is my code:
this is the place where I change my activation function
-----
def conv2d_maxpool(x_tensor, conv_num_outputs, conv_ksize, conv_strides, pool_ksize, pool_strides, layer_name):
conv_layer = tf.layers.conv2d(x_tensor, conv_num_outputs, kernel_size=conv_ksize, strides=conv_strides, activation=tf.nn.selu, name = layer_name)
conv_layer = tf.layers.max_pooling2d(conv_layer, pool_size=pool_ksize, strides=pool_strides)
return conv_layer
-----
graph
tf.reset_default_graph()
#### placeholder ####
input_img = tf.placeholder(dtype=tf.float32, shape=(None, img_size, img_size, 3))
y_true = tf.placeholder(dtype=tf.float32, shape=(None, num_class))
keep_prob = tf.placeholder(tf.float32, name="keep_prob")
lr_in = tf.placeholder(dtype = tf.float32, name = 'learning_rate')
conv_ksize = (3,3)
conv_strides = (1,1)
pool_ksize = (2,2)
pool_strides = (2,2)
n_filters_1 = 32
n_filters_2 = 64
n_filters_3 = 128
n_filters_4 = 256
onebyone_ksize = (1,1)
#CNN
conv_1 = conv2d_maxpool(input_img, n_filters_1, conv_ksize, conv_strides, pool_ksize, pool_strides, layer_name = "conv1")
# conv_1 = tf.layers.conv2d(conv_1, conv_num_outputs, kernel_size=conv_ksize, strides=conv_strides, activation=tf.nn.relu)
# conv_1_norm = tf.layers.batch_normalization(conv_1, name = "batch_norm1")
# conv_1_dropout = tf.layers.dropout(conv_1_norm, rate = keep_prob, training = True, name = "dropout1")
conv_2 = conv2d_maxpool(conv_1, n_filters_2, conv_ksize, conv_strides, pool_ksize, pool_strides, layer_name = "conv2")
# conv_2_norm = tf.layers.batch_normalization(conv_2)
conv_3 = conv2d_maxpool(conv_2, n_filters_3, conv_ksize, conv_strides, pool_ksize, pool_strides, layer_name = "conv3")
# conv_3_norm = tf.layers.batch_normalization(conv_3, name = "batch_norm3")
# conv_3_dropout = tf.layers.dropout(conv_3_norm, rate = keep_prob, training = True, name = "dropout3")
conv_4 = conv2d_maxpool(conv_3, n_filters_4, conv_ksize, conv_strides, pool_ksize, pool_strides, layer_name = "conv4")
flatten = tf.layers.flatten(conv_4)
fc1 = tf.layers.dense(flatten, 256, activation = tf.nn.relu)
out = tf.layers.dense(fc1, 6, activation=None, name= "logits") #logit
predict = tf.nn.softmax(out)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = out, labels = y_true))
optimizer = tf.train.AdamOptimizer(lr).minimize(cost)
##accuracy
correct_pred = tf.equal(tf.argmax(out, 1), tf.argmax(y_true, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32), name='accuracy')
Training
#history/record
train_loss, train_acc = [], []
valid_loss, valid_acc = [], []
update_per_epoch = int(np.floor(X_train.shape[0] / batch_size))
## early stopping and learning rate congig
es_patience = 10
es_n = 0
lr_patience = 3
lr_n = 0
save_model_path = './save'
saver = tf.train.Saver()
sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)
# Initializing the variables
batch_gen = img_gen.flow(generator_input(X_train), y_train, batch_size = 32)
val_batch_gen = img_gen.flow(generator_input(X_valid), y_valid, batch_size = len(X_valid))
for i in range(epoch):
epoch_loss = 0
epoch_acc = 0
for j in range(update_per_epoch):
image, label = next(batch_gen)
_, this_loss, this_acc = sess.run([optimizer, cost, accuracy], feed_dict={
input_img : image,
y_true : label,
lr_in: lr,
keep_prob : keep_probability
})
epoch_loss += this_loss
epoch_acc += this_acc
## end of epoch
epoch_loss /= update_per_epoch
epoch_acc /= update_per_epoch
train_loss.append(epoch_loss)
train_acc.append(epoch_acc)
print('Epoch {:>2} Loss: {:>4.4f} Training Accuracy: {:.6f}'.format(i + 1, epoch_loss, epoch_acc))
valid_image, valid_label = next(val_batch_gen)
valid_this_loss, valid_this_acc = sess.run([cost, accuracy], feed_dict = {
input_img: valid_image,
y_true: valid_label,
lr_in: lr,
keep_prob: 1.
})
valid_loss.append(valid_this_loss)
valid_acc.append(valid_this_acc)
print('Epoch {:>2} Loss: {:>4.4f} Validation Accuracy: {:.6f}'.format(i + 1,valid_this_loss, valid_this_acc))
# early stop
if valid_this_loss > np.min(valid_loss):
es_n += 1
lr_n += 1
else:
es_n = 0
lr_n = 0
saver.save(sess, os.path.join(os.getcwd(), 'bestsession.ckpt'))
# early stop
if es_n >= es_patience:
print("-----------early stopping-------------")
break
# adaptive learning rate
if lr_n >= lr_patience:
lr *= lr_decay_rate
lr_n = 0
print("-----------adjust learning rate------------")
# Save Model
save_path = saver.save(sess, save_model_path)
print('-----model save ------')
----------- 18/09/07------------
I can always reproduce the same result.
And this is my code, I wrote it in Jupyter. But sorry I can't upload the training data:
https://drive.google.com/open?id=1uUE32KrNmWnhLbV8z-fyHSMu6zGCCG_e

Neural network showing same result for different inputs

I am new to tensorflow. I am training my cnn model for digit recognition using tensorflow. Although, it is showing accuracy of 99% during training. But it keeps on displaying same result when tested on my image set e.g. sometimes 8 for all images, sometimes 2 for all images, sometimes 3 for all images.
Please help!
I am attaching code for reference
File trainer.py:
import tensorflow as tf
import numpy as np
from tensorflow.examples.tutorials.mnist import input_data
def layer(input,weight_shape,bias_shape):
w_stddev = (2.0/weight_shape[0])**0.5
w_init = tf.random_normal_initializer(stddev = w_stddev)
b_init = tf.constant_initializer(0)
W = tf.get_variable('W',weight_shape,initializer = w_init)
b = tf.get_variable('B',bias_shape,initializer = b_init)
output = tf.matmul(input,W)+b
return tf.nn.relu(output)
def conv2d(input,weight_shape,bias_shape):
inp = weight_shape[0]*weight_shape[1]*weight_shape[2]
w_init = tf.random_normal_initializer(stddev = (2.0/inp)**0.5)
W = tf.get_variable('W',weight_shape,initializer = w_init)
b_init = tf.constant_initializer(0)
b = tf.get_variable('b',bias_shape,initializer = b_init)
conv_out = tf.nn.conv2d(input,W,strides = [1,1,1,1], padding = 'SAME')
return tf.nn.relu(tf.nn.bias_add(conv_out,b))
def max_pool(input,k=2):
return tf.nn.max_pool(input,ksize = [1,k,k,1], strides = [1,k,k,1],padding = 'SAME')
def inference(x,keep_prob):
x = tf.reshape(x,shape = [-1,28,28,1])
keep_prob = keep_prob[0]
with tf.variable_scope("conv_1"):
conv_1 = conv2d(x,[5,5,1,32],[32])
pool_1 = max_pool(conv_1)
with tf.variable_scope("conv_2"):
conv_2 = conv2d(pool_1,[5,5,32,64],[64])
pool_2 = max_pool(conv_2)
with tf.variable_scope("fc"):
pool_2_flat = tf.reshape(pool_2,[-1,7*7*64])
fc_1 = layer(pool_2_flat,[7*7*64,1024],[1024])
fc_1_drop = tf.nn.dropout(fc_1,keep_prob)
with tf.variable_scope("output"):
output = layer(fc_1_drop,[1024,10],[10])
return output
def evaluate(output,y):
correct_prediction = tf.equal(tf.argmax(output,1),tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction,tf.float32))
return accuracy
def training(cost,global_step):
tf.summary.scalar("cost",cost)
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
train_op = optimizer.minimize(cost,global_step = global_step)
return train_op
def loss(output,y):
xentropy = tf.nn.softmax_cross_entropy_with_logits(labels = y,logits = output)
loss = tf.reduce_mean(xentropy)
return loss
learning_rate = 0.01
display_step = 1
batch_size = 100
training_epoch = 2
mnist = input_data.read_data_sets('MNIST_data/',one_hot = True)
with tf.Graph().as_default():
x = tf.placeholder('float',[None,784],name="x")
y = tf.placeholder('float',[None,10],name="y")
keep_prob = tf.placeholder('float',[1],name="keep_prob")
global_step = tf.Variable(0,name = 'global_step',trainable = False)
output = inference(x,keep_prob)
#print(output.name)
cost_op = loss(output,y)
eval_op = evaluate(output,y)
print(eval_op.name)
train_op = training(cost_op,global_step)
summary_op = tf.summary.merge_all()
saver = tf.train.Saver()
sess = tf.Session()
writer = tf.summary.FileWriter('logistics_logs/',graph_def = sess.graph_def)
init_op = tf.global_variables_initializer()
sess.run(init_op)
for epoch in range(training_epoch):
avg_cost = 0.
total_batch = int(mnist.train.num_examples/batch_size)
for i in range(total_batch):
mbatch_x,mbatch_y = mnist.train.next_batch(batch_size)
feed_dict = {
x: mbatch_x,
y: mbatch_y,
keep_prob: np.asarray([0.5])
}
sess.run(train_op,feed_dict = feed_dict)
mbatch_cost = sess.run(cost_op,feed_dict = feed_dict)
avg_cost += mbatch_cost/total_batch
if epoch % display_step == 0:
val_feed_dict = {
x: mnist.validation.images,
y: mnist.validation.labels,
keep_prob: np.asarray([1])
}
accuracy = sess.run(eval_op,feed_dict = val_feed_dict)
print(epoch+1,'Validation Accuracy : ',accuracy*100,'%')
summary_str = sess.run(summary_op,feed_dict = val_feed_dict)
writer.add_summary(summary_str,sess.run(global_step))
saver.save(sess,'logistics_logs/model_checkpoint/',global_step)
print('Optimization finished')
test_feed_dict = {
x: mnist.test.images,
y: mnist.test.labels,
keep_prob: np.asarray([1])
}
accuracy = sess.run(eval_op,feed_dict = test_feed_dict)
print('Test Accuracy : ',accuracy*100,'%')
File model.py:
import tensorflow as tf
import numpy as np
import os
import cv2
from PIL import Image
from scipy import misc
#img = Image.open('character.bmp')
#a = np.asarray(img).flatten()
#test_data = np.subtract(a,0.,dtype=np.float32)
#test_data = np.reshape(test_data,(784))
#test_data = 255*test_data
#print(test_data)
#print(test_data.shape)
image = cv2.imread('character.jpg')
image = cv2.cvtColor(image,cv2.COLOR_BGR2GRAY)
image = cv2.resize(image,(28,28),cv2.INTER_LINEAR)
images= np.array(image,dtype=np.uint8)
images=images.astype('float32')
images=np.multiply(images,1.0/255.0)
images = np.asarray(images).flatten()
test_data=[images]
#print(test_data)
restore_path = os.path.join(os.path.abspath('./'),"logistics_logs\\model_checkpoint\\")
meta_file_location = os.path.join(restore_path,"-550.meta")
sess = tf.Session()
saver = tf.train.import_meta_graph(meta_file_location)
saver.restore(sess,tf.train.latest_checkpoint(restore_path))
graph = tf.get_default_graph()
x = graph.get_tensor_by_name("x:0")
keep_prob = graph.get_tensor_by_name("keep_prob:0")
feed_dict = {x:test_data,keep_prob:np.asarray([1])}
op = graph.get_tensor_by_name("output/Relu:0")
digit = sess.run(tf.argmax(op,1),feed_dict = feed_dict)
print(int(digit))

Although I'm not quite following your code from one file to the other easily, I'm pretty sure your problem is here:
init_op = tf.global_variables_initializer()
sess.run(init_op)
I suspect that code is being run after you load the network with this code:
saver = tf.train.import_meta_graph(meta_file_location)
saver.restore(sess,tf.train.latest_checkpoint(restore_path))
If that's the case what you're doing is overwriting your model with a random initialization. You should not run the init op when you load the model from disk.
An output of a single value regardless of input is a common result from a randomly initialized network, which is what leads me to this suspicion.

TensorFlow image classification

I am very new to TensorFlow. I am doing the image classification using my own training database.
However, after I trained my own dataset, I have no idea on how to classify the input image.
Here is my code for preparing my own dataset
filenames = ['01.jpg', '02.jpg', '03.jpg', '04.jpg']
label = [0,1,1,1]
filename_queue = tf.train.string_input_producer(filenames)
reader = tf.WholeFileReader()
filename, content = reader.read(filename_queue)
image = tf.image.decode_jpeg(content, channels=3)
image = tf.cast(image, tf.float32)
resized_image = tf.image.resize_images(image, 224, 224)
image_batch , label_batch= tf.train.batch([resized_image,label], batch_size=8, num_threads = 3, capacity=5000)
Is this a correct code for training the dataset?
Afterwards, I try to use it to classify the input images with the following code.
test = ['test.jpg', 'test2.jpg']
test_queue=tf.train.string_input_producer(test)
reader = tf.WholeFileReader()
testname, test_content = reader.read(test_queue)
test = tf.image.decode_jpeg(test_content, channels=3)
test = tf.cast(test, tf.float32)
resized_image = tf.image.resize_images(test, 224,224)
with tf.Session() as sess:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
res = sess.run(resized_image)
coord.request_stop()
coord.join(threads)
However, it does not return the predicted label for the input images.
I am looking for someone to teach me how to classify the images by using my own dataset.
Thank you.

maybe you could try this after you have install PIL python lib:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import time
import math
import numpy
import numpy as np
import random
from PIL import Image
from six.moves import xrange # pylint: disable=redefined-builtin
import tensorflow as tf
# Basic model parameters as external flags.
flags = tf.app.flags
FLAGS = flags.FLAGS
flags.DEFINE_float('learning_rate', 0.01, 'Initial learning rate.')
flags.DEFINE_integer('max_steps', 2000, 'Number of steps to run trainer.')
flags.DEFINE_integer('hidden1', 128, 'Number of units in hidden layer 1.')
flags.DEFINE_integer('hidden2', 32, 'Number of units in hidden layer 2.')
flags.DEFINE_integer('batch_size', 4, 'Batch size. '
'Must divide evenly into the dataset sizes.')
flags.DEFINE_string('train_dir', 'data', 'Directory to put the training data.')
flags.DEFINE_boolean('fake_data', False, 'If true, uses fake data '
'for unit testing.')
NUM_CLASSES = 2
IMAGE_SIZE = 28
CHANNELS = 3
IMAGE_PIXELS = IMAGE_SIZE * IMAGE_SIZE * CHANNELS
def inference(images, hidden1_units, hidden2_units):
# Hidden 1
with tf.name_scope('hidden1'):
weights = tf.Variable(
tf.truncated_normal([IMAGE_PIXELS, hidden1_units],
stddev=1.0 / math.sqrt(float(IMAGE_PIXELS))),
name='weights')
biases = tf.Variable(tf.zeros([hidden1_units]),
name='biases')
hidden1 = tf.nn.relu(tf.matmul(images, weights) + biases)
# Hidden 2
with tf.name_scope('hidden2'):
weights = tf.Variable(
tf.truncated_normal([hidden1_units, hidden2_units],
stddev=1.0 / math.sqrt(float(hidden1_units))),
name='weights')
biases = tf.Variable(tf.zeros([hidden2_units]),
name='biases')
hidden2 = tf.nn.relu(tf.matmul(hidden1, weights) + biases)
# Linear
with tf.name_scope('softmax_linear'):
weights = tf.Variable(
tf.truncated_normal([hidden2_units, NUM_CLASSES],
stddev=1.0 / math.sqrt(float(hidden2_units))),
name='weights')
biases = tf.Variable(tf.zeros([NUM_CLASSES]),
name='biases')
logits = tf.matmul(hidden2, weights) + biases
return logits
def cal_loss(logits, labels):
labels = tf.to_int64(labels)
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits, labels, name='xentropy')
loss = tf.reduce_mean(cross_entropy, name='xentropy_mean')
return loss
def training(loss, learning_rate):
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
global_step = tf.Variable(0, name='global_step', trainable=False)
train_op = optimizer.minimize(loss, global_step=global_step)
return train_op
def evaluation(logits, labels):
correct = tf.nn.in_top_k(logits, labels, 1)
return tf.reduce_sum(tf.cast(correct, tf.int32))
def placeholder_inputs(batch_size):
images_placeholder = tf.placeholder(tf.float32, shape=(batch_size,IMAGE_PIXELS))
labels_placeholder = tf.placeholder(tf.int32, shape=(batch_size))
return images_placeholder, labels_placeholder
def fill_feed_dict(images_feed,labels_feed, images_pl, labels_pl):
feed_dict = {
images_pl: images_feed,
labels_pl: labels_feed,
}
return feed_dict
def do_eval(sess,
eval_correct,
images_placeholder,
labels_placeholder,
data_set):
# And run one epoch of eval.
true_count = 0 # Counts the number of correct predictions.
steps_per_epoch = 4 // FLAGS.batch_size
num_examples = steps_per_epoch * FLAGS.batch_size
for step in xrange(steps_per_epoch):
feed_dict = fill_feed_dict(train_images,train_labels,
images_placeholder,
labels_placeholder)
true_count += sess.run(eval_correct, feed_dict=feed_dict)
precision = true_count / num_examples
print(' Num examples: %d Num correct: %d Precision # 1: %0.04f' %
(num_examples, true_count, precision))
# Get the sets of images and labels for training, validation, and
train_images = []
for filename in ['01.jpg', '02.jpg', '03.jpg', '04.jpg']:
image = Image.open(filename)
image = image.resize((IMAGE_SIZE,IMAGE_SIZE))
train_images.append(np.array(image))
train_images = np.array(train_images)
train_images = train_images.reshape(4,IMAGE_PIXELS)
label = [0,1,1,1]
train_labels = np.array(label)
def run_training():
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# Generate placeholders for the images and labels.
images_placeholder, labels_placeholder = placeholder_inputs(4)
# Build a Graph that computes predictions from the inference model.
logits = inference(images_placeholder,
FLAGS.hidden1,
FLAGS.hidden2)
# Add to the Graph the Ops for loss calculation.
loss = cal_loss(logits, labels_placeholder)
# Add to the Graph the Ops that calculate and apply gradients.
train_op = training(loss, FLAGS.learning_rate)
# Add the Op to compare the logits to the labels during evaluation.
eval_correct = evaluation(logits, labels_placeholder)
# Create a saver for writing training checkpoints.
saver = tf.train.Saver()
# Create a session for running Ops on the Graph.
sess = tf.Session()
# Run the Op to initialize the variables.
init = tf.initialize_all_variables()
sess.run(init)
# And then after everything is built, start the training loop.
for step in xrange(FLAGS.max_steps):
start_time = time.time()
feed_dict = fill_feed_dict(train_images,train_labels,
images_placeholder,
labels_placeholder)
_, loss_value = sess.run([train_op, loss],
feed_dict=feed_dict)
duration = time.time() - start_time
if step % 100 == 0:
# Print status to stdout.
print('Step %d: loss = %.2f (%.3f sec)' % (step, loss_value, duration))
if (step + 1) % 1000 == 0 or (step + 1) == FLAGS.max_steps:
saver.save(sess, FLAGS.train_dir, global_step=step)
print('Training Data Eval:')
do_eval(sess,
eval_correct,
images_placeholder,
labels_placeholder,
train_images)
def main(_):
run_training()
if __name__ == '__main__':
tf.app.run()