I have a CNN for CIFAR-10 dataset with layers:
[IN] -> [CONV] -> [POOL] -> [CONV] -> [POOL] -> [FC] -> [DROPOUT] -> [LOGITS] -> [OUT]
\-> [EMBEDDINGS]
Estimator code:
config = tf.contrib.learn.RunConfig(save_checkpoints_secs=30)
# Create the Estimator
classifier = tf.estimator.Estimator(model_fn=inference, config=config, model_dir=LOG_DIR)
train_images, train_labels, train_labels_onehot = Utils.load_training_data()
hooks = [
# logging hook
tf.train.LoggingTensorHook(tensors=tensors_to_log, every_n_iter=50),
]
train_input_fn = tf.estimator.inputs.numpy_input_fn(x={'x': train_images}, y=train_labels)
classifier.train(input_fn=train_input_fn, steps=FLAGS.steps, hooks=hooks)
Inference function code:
def inference(self, features, labels, mode):
try:
images = tf.cast(features['x'], tf.float32)
# Input Layer
with tf.name_scope('Data'):
input_layer = tf.reshape(images, [-1, img_width, img_height, num_channels])
# Convolutional Layer 1
with tf.variable_scope('ConvLayer1'):
conv1 = tf.layers.conv2d(inputs=input_layer, filters=32, kernel_size=[5, 5],
padding="same", activation=tf.nn.relu)
pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2)
logging.info('Convolutional Layer 1 build successful..')
# Convolutional Layer 1
with tf.variable_scope('ConvLayer2'):
conv2 = tf.layers.conv2d(inputs=pool1, filters=64, kernel_size=[5, 5],
padding="same", activation=tf.nn.relu)
pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2)
logging.info('Convolutional Layer 2 build successful..')
# Fully Connected Layer
with tf.variable_scope('FullyConnectedLayer'):
pool2_flat = tf.reshape(pool2, [-1, 8 * 8 * 64])
dense = tf.layers.dense(inputs=pool2_flat, units=1024, activation=tf.nn.relu)
dropout = tf.layers.dropout(inputs=dense, rate=0.4,
training=(mode == tf.estimator.ModeKeys.TRAIN))
logging.info('Fully Connected Layer build successful..')
tf.summary.histogram('dropout', dropout)
# Logits Layer
logits = tf.layers.dense(inputs=dropout, units=10)
tf.summary.histogram('logits', logits)
logging.info('Logits Layer build successful..')
predictions = {
# Generate predictions (for PREDICT and EVAL mode)
"classes": tf.argmax(input=logits, axis=1),
# Add `softmax_tensor` to the graph. It is used for PREDICT and by the
# `logging_hook`.
"probabilities": tf.nn.softmax(logits, name="softmax_tensor")
}
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions,
evaluation_hooks=[])
# Calculate Loss (for both TRAIN and EVAL modes)
onehot_labels = tf.one_hot(indices=tf.cast(labels, tf.int32), depth=10)
loss = tf.losses.softmax_cross_entropy(onehot_labels=onehot_labels,
logits=logits)
tf.summary.histogram('loss', loss)
logging.info('Losses build successful..')
# Configure the Training Op (for TRAIN mode)
if mode == tf.estimator.ModeKeys.TRAIN:
learning_rate = tf.train.exponential_decay(start_learning_rate,
tf.train.get_global_step(), 1000, 0.9, staircase=True)
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
train_op = optimizer.minimize(loss=loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op,
scaffold=tf.train.Scaffold(
summary_op=tf.summary.merge_all(),
))
# Add evaluation metrics (for EVAL mode)
accuracy = tf.metrics.accuracy(labels=labels, predictions=predictions["classes"])
tf.summary.histogram('accuracy', accuracy)
logging.info('Accuracy metric build successful..')
return tf.estimator.EstimatorSpec(mode=mode, loss=loss,
train_op=train_op,
scaffold=tf.train.Scaffold(
summary_op=tf.summary.merge_all()
))
I am trying to work with Embeddings Visualization in tensorflow and here I want to visualize dropout output as embeddings.
Code that I've found to use embeddings:
sess = tf.InteractiveSession()
# Input set for Embedded TensorBoard visualization
# Performed with cpu to conserve memory and processing power
with tf.device("/cpu:0"):
embedding = tf.Variable(self._data, trainable=False, name='embedding')
sess.run(embedding.initializer)
writer = tf.summary.FileWriter(LOG_DIR + '/projector', sess.graph)
config = projector.ProjectorConfig()
embed = config.embeddings.add()
embed.tensor_name = embedding.name
embed.metadata_path = os.path.join(LOG_DIR + '/projector/metadata.tsv')
embed.sprite.image_path = os.path.join(DATA_DIR + '/cifar_10k_sprite.png')
embed.sprite.single_image_dim.extend([img_width, img_height])
projector.visualize_embeddings(writer, config)
saver = tf.train.Saver([embedding])
saver.save(sess, os.path.join(LOG_DIR, 'projector/a_model.ckpt'))
It does not work in my case, because I am using Estimator class and I have not access to the session.
Ways that I've tried:
Pass numpy.array variable to the model_fn of Estimator where I can set the value to that variable and then pass that variable to SessionRunHook where I can access session and save data to the file. Did not work because all arguments passed to the Estimator become tensors. So that way does not work cause I'm already have dropout layer tensors.
Create global variable where I can put all values of dropout layer. Did not work too because for that approach I need to access tensors values.
As far as I understood Estimator architecture the main problem is to get dropout layer output signals to the out of the Estimator and pass it somehow to the SessionRunHook to save them as Embeddings. But I think it's not the best way.
What is the right way of using Embeddings in Estimator?
This is how I did it guys (but it may be not the most efficient way):
SessinRunHook:
import tensorflow as tf
from classes.Utils import Utils
class EmbeddingSaverHook(tf.train.SessionRunHook):
def __init__(self, values, labels, captions):
self._saver = None
self._classes = Utils.get_classnames()
self._dense3 = None
self._labels = None
self._emb_values = values
self._emb_labels = labels
self._emb_captions = captions
def begin(self):
self._dense3 = tf.get_default_graph().get_tensor_by_name("dense3/BiasAdd:0")
self._labels = tf.get_default_graph().get_tensor_by_name("labels:0")
def before_run(self, run_context):
return tf.train.SessionRunArgs([self._dense3, self._labels])
def after_run(self, run_context, run_values):
self._emb_values.extend(run_values[0][0])
self._emb_labels.extend(run_values[0][1])
self._emb_captions.extend([self._classes[x] for x in run_values[0][1]])
def end(self, session):
pass
Full code you can see in my github repo
Related
I am new to machine learning and I am trying to make a semantic segmentation model for detecting couches in images. However, my validation loss and training loss are stuck at around 0.25, and my validation binary IoU and training binary IoU are stuck at around 0.7. Here is the code for my model and my training parameters.
def upsample(filters, size, norm_type='batchnorm', apply_dropout=False, kernel_regularizer=None):
"""Upsamples an input.
Conv2DTranspose => Batchnorm => Dropout => Relu
Args:
filters: number of filters
size: filter size
norm_type: Normalization type; either 'batchnorm' or 'instancenorm'.
apply_dropout: If True, adds the dropout layer
Returns:
Upsample Sequential Model
"""
initializer = tf.random_normal_initializer(0., 0.02)
result = tf.keras.Sequential()
result.add(
tf.keras.layers.Conv2DTranspose(filters, size, strides=2,
padding='same',
kernel_initializer=initializer,
kernel_regularizer=kernel_regularizer,
use_bias=False))
if norm_type.lower() == 'batchnorm':
result.add(tf.keras.layers.BatchNormalization())
elif norm_type.lower() == 'instancenorm':
result.add(InstanceNormalization())
if apply_dropout:
result.add(tf.keras.layers.Dropout(0.3))
result.add(tf.keras.layers.ReLU())
return result
class Unet:
output_channels = 1
def __init__(self):
base_model = tf.keras.applications.MobileNetV2(input_shape=[224, 224, 3], include_top=False)
# Use the activations of these layers
layer_names = [
'block_1_expand_relu',
'block_3_expand_relu',
'block_6_expand_relu',
'block_13_expand_relu',
'block_16_project',
]
base_model_outputs = [base_model.get_layer(name).output for name in layer_names]
# Create the feature extraction model
down_stack = tf.keras.Model(inputs=base_model.input, outputs=base_model_outputs)
down_stack.trainable = False
up_stack = [
upsample(1024, 3, norm_type='batchnorm', apply_dropout=True, kernel_regularizer="l2"),
upsample(512, 3, norm_type='batchnorm', apply_dropout=True, kernel_regularizer="l2"),
upsample(256, 3, norm_type='batchnorm', apply_dropout=True, kernel_regularizer="l2"),
upsample(128, 3, norm_type='batchnorm', apply_dropout=False, kernel_regularizer="l2"),
upsample(64, 3, norm_type='batchnorm', apply_dropout=True, kernel_regularizer="l2"),
]
inputs = tf.keras.layers.Input(shape=[224, 224, 3])
# Downsampling through the model
skips = down_stack(inputs)
x = skips[-1]
skips = reversed(skips[:-1])
# Upsampling and establishing the skip connections
for up, skip in zip(up_stack, skips):
x = up(x)
concat = tf.keras.layers.Concatenate()
x = concat([x, skip])
last = tf.keras.layers.Conv2DTranspose(
self.output_channels,
kernel_size=3, strides=2,
padding='same', activation='sigmoid')
x = last(x)
optimizer = tf.keras.optimizers.Nadam(1e-4)
metrics = [tf.keras.metrics.BinaryIoU(target_class_ids=[0, 1], threshold=0.5)]
self.model = tf.keras.Model(inputs=inputs, outputs=x)
self.model.compile(
optimizer=optimizer,
loss=tf.keras.losses.BinaryCrossentropy(),
metrics=metrics)
def train():
batch_size = 4
EPOCHS = 250
VAL_SUBSPLITS = 2
VALIDATION_STEPS = dataset_size // batch_size // VAL_SUBSPLITS
STEPS_PER_EPOCH = train_dataset_size // batch_size
unet.load_weights('./checkpoints/my_checkpoint')
history = unet.fit(train_gen, epochs=EPOCHS,
steps_per_epoch=STEPS_PER_EPOCH,
validation_steps=VALIDATION_STEPS,
validation_data=val_gen,
callbacks=[
ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=4),
EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=False)
],
)
Are there any tips to improve my model?
Current output: https://imgur.com/a/5Jqn1oX
What I've tried:
Augmented my data generator which is based from the COCO dataset
Used an Adam optimizer with an exponential decaying learning rate schedule
Increased batch size from 4 to 32
Used learning rates from 0.01 to 0.0000001
I am trying to implement the VGG but am getting the above odd error. I am running TFv2 on Ubuntu. Could this be because I am not running CUDA?
The code is from here.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
# Imports
import time
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
# tf.logging.set_verbosity(tf.logging.INFO)
from tensorflow.keras.layers import Conv2D, Dense, Flatten
np.random.seed(1)
mnist = tf.keras.datasets.mnist
(train_data, train_labels), (eval_data, eval_labels) = mnist.load_data()
train_data, train_labels = train_data / 255.0, train_labels / 255.0
# Add a channels dimension
train_data = train_data[..., tf.newaxis]
train_labels = train_labels[..., tf.newaxis]
index = 7
plt.imshow(train_data[index].reshape(28, 28))
plt.show()
time.sleep(5);
print("y = " + str(np.squeeze(train_labels[index])))
print ("number of training examples = " + str(train_data.shape[0]))
print ("number of evaluation examples = " + str(eval_data.shape[0]))
print ("X_train shape: " + str(train_data.shape))
print ("Y_train shape: " + str(train_labels.shape))
print ("X_test shape: " + str(eval_data.shape))
print ("Y_test shape: " + str(eval_labels.shape))
print("done")
def cnn_model_fn(features, labels, mode):
# Input Layer
input_height, input_width = 28, 28
input_channels = 1
input_layer = tf.reshape(features["x"], [-1, input_height, input_width, input_channels])
# Convolutional Layer #1 and Pooling Layer #1
conv1_1 = tf.layers.conv2d(inputs=input_layer, filters=64, kernel_size=[3, 3], padding="same",
activation=tf.nn.relu)
conv1_2 = tf.layers.conv2d(inputs=conv1_1, filters=64, kernel_size=[3, 3], padding="same", activation=tf.nn.relu)
pool1 = tf.layers.max_pooling2d(inputs=conv1_2, pool_size=[2, 2], strides=2, padding="same")
# Convolutional Layer #2 and Pooling Layer #2
conv2_1 = tf.layers.conv2d(inputs=pool1, filters=128, kernel_size=[3, 3], padding="same", activation=tf.nn.relu)
conv2_2 = tf.layers.conv2d(inputs=conv2_1, filters=128, kernel_size=[3, 3], padding="same", activation=tf.nn.relu)
pool2 = tf.layers.max_pooling2d(inputs=conv2_2, pool_size=[2, 2], strides=2, padding="same")
# Convolutional Layer #3 and Pooling Layer #3
conv3_1 = tf.layers.conv2d(inputs=pool2, filters=256, kernel_size=[3, 3], padding="same", activation=tf.nn.relu)
conv3_2 = tf.layers.conv2d(inputs=conv3_1, filters=256, kernel_size=[3, 3], padding="same", activation=tf.nn.relu)
pool3 = tf.layers.max_pooling2d(inputs=conv3_2, pool_size=[2, 2], strides=2, padding="same")
# Convolutional Layer #4 and Pooling Layer #4
conv4_1 = tf.layers.conv2d(inputs=pool3, filters=512, kernel_size=[3, 3], padding="same", activation=tf.nn.relu)
conv4_2 = tf.layers.conv2d(inputs=conv4_1, filters=512, kernel_size=[3, 3], padding="same", activation=tf.nn.relu)
pool4 = tf.layers.max_pooling2d(inputs=conv4_2, pool_size=[2, 2], strides=2, padding="same")
# Convolutional Layer #5 and Pooling Layer #5
conv5_1 = tf.layers.conv2d(inputs=pool4, filters=512, kernel_size=[3, 3], padding="same", activation=tf.nn.relu)
conv5_2 = tf.layers.conv2d(inputs=conv5_1, filters=512, kernel_size=[3, 3], padding="same", activation=tf.nn.relu)
pool5 = tf.layers.max_pooling2d(inputs=conv5_2, pool_size=[2, 2], strides=2, padding="same")
# FC Layers
pool5_flat = tf.contrib.layers.flatten(pool5)
FC1 = tf.layers.dense(inputs=pool5_flat, units=4096, activation=tf.nn.relu)
FC2 = tf.layers.dense(inputs=FC1, units=4096, activation=tf.nn.relu)
FC3 = tf.layers.dense(inputs=FC2, units=1000, activation=tf.nn.relu)
"""the training argument takes a boolean specifying whether or not the model is currently
being run in training mode; dropout will only be performed if training is true. here,
we check if the mode passed to our model function cnn_model_fn is train mode. """
dropout = tf.layers.dropout(inputs=FC3, rate=0.4, training=mode == tf.estimator.ModeKeys.TRAIN)
# Logits Layer or the output layer. which will return the raw values for our predictions.
# Like FC layer, logits layer is another dense layer. We leave the activation function empty
# so we can apply the softmax
logits = tf.layers.dense(inputs=dropout, units=10)
# Then we make predictions based on raw output
predictions = {
# Generate predictions (for PREDICT and EVAL mode)
# the predicted class for each example - a vlaue from 0-9
"classes": tf.argmax(input=logits, axis=1),
# to calculate the probablities for each target class we use the softmax
"probabilities": tf.nn.softmax(logits, name="softmax_tensor")
}
# so now our predictions are compiled in a dict object in python and using that we return an estimator object
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)
'''Calculate Loss (for both TRAIN and EVAL modes): computes the softmax entropy loss.
This function both computes the softmax activation function as well as the resulting loss.'''
loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)
# Configure the Training Options (for TRAIN mode)
if mode == tf.estimator.ModeKeys.TRAIN:
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)
train_op = optimizer.minimize(loss=loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)
# Add evaluation metrics (for EVAL mode)
eval_metric_ops = {
"accuracy": tf.metrics.accuracy(labels=labels,
predictions=predictions["classes"])}
return tf.estimator.EstimatorSpec(mode=mode,
loss=loss,
eval_metric_ops=eval_metric_ops)
print("done2")
mnist_classifier = tf.estimator.Estimator(model_fn=cnn_model_fn,
model_dir="/tmp/mnist_vgg13_model")
print("done3")
train_input_fn = tf.compat.v1.estimator.inputs.numpy_input_fn(x={"x": train_data},
y=train_labels,
batch_size=100,
num_epochs=100,
shuffle=True)
print("done4")
mnist_classifier.train(input_fn=train_input_fn,
steps=None,
hooks=None)
print("done5")
eval_input_fn = tf.estimator.inputs.numpy_input_fn(x={"x": eval_data},
y=eval_labels,
num_epochs=1,
shuffle=False)
print("done6")
eval_results = mnist_classifier.evaluate(input_fn=eval_input_fn)
print(eval_results)
The code you're using was written in Tensorflow v1.x, and is not compatible as it is with Tensorflow v2. The easiest solution is probably to downgrade to a version of tensorflow v1 to run the code as it is.
An other option would be to could follow this guide to migrate the code from v1 to v2.
A third option would be to use the tf.compat module to get some retro-compatibility. For example, tf.layers does not exist anymore in Tensorflow v2. You can use tf.compat.v1.layers (see for example the Conv2D function) instead, but this is a temporary fix, as these functions will be removed in a future version.
You can use postfix compat.v1 to make code written for tensorflow 1.x work with newer versions.
In your case this can be achived by changing:
tf.layers.conv2d
to
tf.compat.v1.layers.conv2d
You can read more about migrating tensorflow v1.x to tensorflow v2.x here:
https://www.tensorflow.org/guide/migrate
Use tensorflow 1.x instead of tensorflow 2.x versions. But remember there is no 2.x version on Python 3.8. Use a lower version of Python which has tensorflow 1.x.
python3.6 -m pip install tensorflow==1.8.0
I am trying to train a model in TensorFlow. I've got a problem with the labels. Here is my input function:
def my_input_fn():
filenames = tf.constant(glob.glob("C:/test_proje/*.jpg"))
labels = tf.constant([0, 0, 1, 1, 1, 1, 1, 0, 0, 0])
labels = tf.one_hot(labels, 2)
dataset = tf.data.Dataset.from_tensor_slices((filenames, labels))
dataset = dataset.map(_parse_function)
return dataset
And here is the CNN model
def cnn_model_fn(features, labels, mode):
"""Model function for CNN."""
print(labels.shape)
print(labels[0])
# Input Layer
input_layer = tf.reshape(features["image"], [-1, 168, 84, 3])
# Convolutional Layer #1
conv1 = tf.layers.conv2d(
inputs=input_layer,
filters=32,
kernel_size=[5, 5],
padding="same",
activation=tf.nn.relu)
# Pooling Layer #1
pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2],
strides=2)
# Convolutional Layer #2 and Pooling Layer #2
conv2 = tf.layers.conv2d(
inputs=pool1,
filters=64,
kernel_size=[5, 5],
padding="same",
activation=tf.nn.relu)
pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2],
strides=2)
# Dense Layer
pool2_flat = tf.reshape(pool2, [-1, 42 * 21 * 64])
dense = tf.layers.dense(inputs=pool2_flat, units=4,
activation=tf.nn.relu)
dropout = tf.layers.dropout(
inputs=dense, rate=0.4, training=mode ==
tf.estimator.ModeKeys.TRAIN)
# Logits Layer
logits = tf.layers.dense(inputs=dropout, units=2)
predictions = {
# Generate predictions (for PREDICT and EVAL mode)
"classes": tf.argmax(input=logits, axis=1),
# Add `softmax_tensor` to the graph. It is used for PREDICT and by
the
# `logging_hook`.
"probabilities": tf.nn.softmax(logits, name="softmax_tensor")
}
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)
# Calculate Loss (for both TRAIN and EVAL modes)
loss = tf.losses.sparse_softmax_cross_entropy(labels=labels,
logits=logits)
# Configure the Training Op (for TRAIN mode)
if mode == tf.estimator.ModeKeys.TRAIN:
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)
train_op = optimizer.minimize(
loss=loss,
global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode=mode, loss=loss,
train_op=train_op)
# Add evaluation metrics (for EVAL mode)
eval_metric_ops = {
"accuracy": tf.metrics.accuracy(
labels=labels, predictions=predictions["classes"])}
return tf.estimator.EstimatorSpec(
mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)
I am getting this error: ValueError: Shape mismatch: The shape of labels (received (2,)) should equal the shape of logits except for the last dimension (received (1, 2)).
When I print the shape of labels before cnn_model_fn, it is (10,2). But when I print it in cnn_model_fn, it suddenly becomes (2,)
Thanks.
I think it's your use of sparse_softmax_cross_entropy. You feed it one hot encoded labels which it doesn't want.
Switch to just normal softmax_cross_entropy and see if that works.
https://stackoverflow.com/a/37317322/7431458
I,ve followed the tutorial in tensorflow tutorial to build a MNIST model for hand written digits recognition. I want to test the model by inputing a single image to the classifier and get the output it predicted.
Here is the complete code of the classifier.
I've tried by reading the image using imread but it didn't worked
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import numpy as np
import tensorflow as tf
import pylab
import os
from scipy.misc import imread
import matplotlib.image as mpimg
import matplotlib.pyplot as plt
tf.logging.set_verbosity(tf.logging.INFO)
def cnn_model_fn(features, labels, mode):
input_layer = tf.reshape(features["x"],[-1,28,28,1])
conv1 = tf.layers.conv2d(
inputs=input_layer,
filters=32,
kernel_size=[5,5],
padding = "same",
activation = tf.nn.relu)
pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2,2], strides=2)
conv2 = tf.layers.conv2d(
inputs=pool1,
filters=64,
kernel_size=5,
padding="same",
activation = tf.nn.relu)
pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2,2], strides=2)
pool2_flat = tf.reshape(pool2, [ -1, 7 * 7 * 64 ] )
dense = tf.layers.dense(inputs=pool2_flat, units=1024, activation=tf.nn.relu)
dropout = tf.layers.dropout(inputs=dense, rate = 0.4, training=mode == tf.estimator.ModeKeys.TRAIN)
logits = tf.layers.dense(inputs=dropout, units=10)
predictions = {
"classes":tf.argmax(input=logits, axis=1),
"probabilites": tf.nn.softmax(logits, name="softmax_tensor")
}
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)
loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)
if mode == tf.estimator.ModeKeys.TRAIN:
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)
train_op = optimizer.minimize(
loss=loss,
global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)
eval_metric_ops={
"accuracy":tf.metrics.accuracy(
labels=labels, predictions=predictions["classes"])}
return tf.estimator.EstimatorSpec(mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)
def main(unused_argv):
# Load training and eval data
mnist = tf.contrib.learn.datasets.load_dataset("mnist")
train_data = mnist.train.images # Returns np.array
train_labels = np.asarray(mnist.train.labels, dtype=np.int32)
eval_data = mnist.test.images # Returns np.array
eval_labels = np.asarray(mnist.test.labels, dtype=np.int32)
# Create the Estimator
mnist_classifier = tf.estimator.Estimator(
model_fn=cnn_model_fn, model_dir="/tmp/mnist_convnet_model")
Set up logging for predictions
Log the values in the "Softmax" tensor with label "probabilities"
tensors_to_log = {"probabilities": "softmax_tensor"}
logging_hook = tf.train.LoggingTensorHook(
tensors=tensors_to_log, every_n_iter=50)
# Train the model
train_input_fn = tf.estimator.inputs.numpy_input_fn(
x={"x": train_data},
y=train_labels,
batch_size=100,
num_epochs=None,
shuffle=True)
mnist_classifier.train(
input_fn=train_input_fn,
steps=20000,
hooks=[logging_hook])
# Evaluate the model and print results
eval_input_fn = tf.estimator.inputs.numpy_input_fn(
x={"x": eval_data},
y=eval_labels,
num_epochs=1,
shuffle=False)
eval_results = mnist_classifier.evaluate(input_fn=eval_input_fn)
print(eval_results)
A resource I found really helpful was Tensorflow's official tutorial, which includes a demo of their Estimator. In it, you can find how to test numpy arrays with the model. Here's the link: https://github.com/tensorflow/models/tree/master/official/mnist
Hope this helps!
I'm trying to build a simple net of 2 input neurons (+1 bias) going into 1 output neuron to teach it the "and"-function. It's based on the mnist-clissification example, so it might be overly complex for the task, but it's about the general structure of such nets for me, so please don't say "you can just do it in numpy" or something, it's about tensorflow NNs for me. So here is the code:
import tensorflow as tf
import numpy as np
tf.logging.set_verbosity(tf.logging.INFO)
def model_fn(features, labels, mode):
input_layer = tf.reshape(features["x"], [-1, 2])
output_layer = tf.layers.dense(inputs=input_layer, units=1, activation=tf.nn.relu, name="output_layer")
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode, predictions=output_layer)
loss = tf.losses.mean_squared_error(labels=labels, predictions=output_layer)
if mode == tf.estimator.ModeKeys.TRAIN:
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)
train_op = optimizer.minimize(loss=loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)
eval_metrics_ops = {"accuracy": tf.metrics.accuracy(labels=labels, predictions=output_layer)}
return tf.estimator.EstimatorSpec(mode=mode, predictions=output_layer, loss=loss)
def main(unused_arg):
train_data = np.asarray(np.reshape([[0,0],[0,1],[1,0],[1,1]],[4,2]))
train_labels = np.asarray(np.reshape([0,0,0,1],[4,1]))
eval_data = train_data
eval_labels = train_labels
classifier = tf.estimator.Estimator(model_fn=model_fn, model_dir="/tmp/NN_AND")
tensors_to_log = {"The output:": "output_layer"}
logging_hook = tf.train.LoggingTensorHook(tensors=tensors_to_log,every_n_iter=10)
train_input_fn = tf.estimator.inputs.numpy_input_fn(x={"x":train_data}, y=train_labels, batch_size=10, num_epochs=None, shuffle=True)
classifier.train(input_fn=train_input_fn, steps=2000, hooks=[logging_hook])
eval_input_fn = tf.estimator.inputs.numpy_input_fn(x={"x":eval_data}, y=eval_labels, batch_size=1, shuffle=False)
eval_results = classifier.evaluate(input_fn=eval_input_fn)
print(eval_results)
if __name__ == "__main__":
tf.app.run()
I've made few slight modifications to your code which enable learning the and function:
1) change your train_data to float32 representation.
train_data = np.asarray(np.reshape([[0,0],[0,1],[1,0],[1,1]],[4,2]), dtype=np.float32)`
2) Remove relu activation from the output layer - generally speaking, using relus in the output layer is not recommended. This might lead to dead relus and all gradients will be equal zero, which in turn will not make any learning possible.
output_layer = tf.layers.dense(inputs=input_layer, units=1, activation=None, name="output_layer")
3) In your eval_metrics_ops make sure you round the result so you can actually measure accuracy:
eval_metrics_ops = {"accuracy": tf.metrics.accuracy(labels=labels, predictions=tf.round(output_layer))}
4) Don't fortget to add the eval_metrics_ops parameter you defined to the estimator:
return tf.estimator.EstimatorSpec(mode=mode, predictions=output_layer, loss=loss, eval_metric_ops=eval_metrics_ops)
In addition, to log the last layer output you should use:
tensors_to_log = {"The output:": "output_layer/BiasAdd:0"}