We Keep Coding

Keras: one-hot for labels in `image_dataset_from_directory`

I am trying to do a binary image classification using efficientNet. The following is my code.
import matplotlib.pyplot as plt
from tensorflow.keras.applications import EfficientNetB0
from tensorflow.keras.preprocessing import image_dataset_from_directory
from tensorflow.keras.models import Sequential
from tensorflow.keras import layers
from tensorflow.data import AUTOTUNE
DATA_DIR = "img/"
IMG_SIZE = 224
NUM_CLASSES = 2
EPOCH = 50
def train_val_split(DATA_DIR, IMG_SIZE):
val_data = image_dataset_from_directory(
DATA_DIR,
labels="inferred",
label_mode="binary",
color_mode="rgb",
batch_size=32,
image_size=(IMG_SIZE, IMG_SIZE),
validation_split=0.2,
subset="training",
seed=1
)
train_data = image_dataset_from_directory(
DATA_DIR,
labels="inferred",
label_mode="binary",
color_mode="rgb",
batch_size=32,
image_size=(IMG_SIZE, IMG_SIZE),
validation_split=0.2,
subset="validation",
seed=1
)
train_data = train_data.cache().prefetch(buffer_size=AUTOTUNE)
val_data = val_data.cache().prefetch(buffer_size=AUTOTUNE)
return train_data, val_data
def model_arch(NUM_CLASSES, IMG_SIZE):
"""efficientnet transfer learning"""
inputs = layers.Input(shape=(IMG_SIZE, IMG_SIZE, 3))
img_augmentation = Sequential(
[
layers.RandomRotation(factor=0.15),
layers.RandomTranslation(height_factor=0.1, width_factor=0.1),
layers.RandomFlip(),
layers.RandomContrast(factor=0.1),
],
name="img_augmentation",
)
x = img_augmentation(inputs)
# model = EfficientNetB0(include_top=False, input_tensor=x, weights="imagenet")
model = EfficientNetB0(include_top=False, input_tensor=x, weights='model/efficientnetb0_notop.h5')
# Freeze the pretrained weights
model.trainable = False
# Rebuild top
x = layers.GlobalAveragePooling2D(name="avg_pool")(model.output)
x = layers.BatchNormalization()(x)
top_dropout_rate = 0.2
x = layers.Dropout(top_dropout_rate, name="top_dropout")(x)
outputs = layers.Dense(NUM_CLASSES, activation="softmax", name="pred")(x)
# Compile
model = tf.keras.Model(inputs, outputs, name="EfficientNet")
optimizer = tf.keras.optimizers.Adam(learning_rate=1e-2)
model.compile(
optimizer=optimizer,
loss="binary_crossentropy",
metrics=["accuracy"]
)
return model
if __name__ == "__main__":
train_data, val_data = train_val_split(DATA_DIR, IMG_SIZE)
model = model_arch(NUM_CLASSES, IMG_SIZE)
hist = model.fit(train_data,
epochs=EPOCH,
validation_data=val_data,
verbose=1)
However, I encountered the following error.
ValueError: logits and labels must have the same shape ((None, 2) vs (None, 1))
I found out that this is because the labels loaded in image_dataset_from_directory is not one-hot encoded.
print(train_data)
<PrefetchDataset shapes: ((None, 224, 224, 3), (None, 1)), types: (tf.float32, tf.float32)>
How can I tweak the code for train_data and val_data so that it can fit into the model without issues?
Thanks.

Managed to figure out the answer!
import tensorflow as tf
# one-hot encoding
train_data = train_data.map(lambda x, y: (x, tf.one_hot(y, depth=NUM_CLASSES)))
val_data = val_data.map(lambda x, y: (x, tf.one_hot(y, depth=NUM_CLASSES)))

You need to change label_mode to a better option.
In your case, I think you want label_mode='categorical',.
As #Dr. Snoopy said the information is here: https://www.tensorflow.org/api_docs/python/tf/keras/utils/image_dataset_from_directory

Tensorflow keras: Problem with loading the weights of the optimizer

I have run the base model to a good accuracy and now i want to load these weights and use them for a model with a few additional layers and later for hyperparameter tuning.
First i construct this new model
input_tensor = Input(shape=train_generator.image_shape)
base_model = applications.ResNet152(weights='imagenet', include_top=False, input_tensor=input_tensor)
for layer in base_model.layers[:]:
layer.trainable = False
x = Flatten()(base_model.output)
x = Dense(1024, kernel_regularizer=tf.keras.regularizers.L2(l2=0.01),
kernel_initializer=tf.keras.initializers.HeNormal(), kernel_constraint=tf.keras.constraints.UnitNorm(axis=0))(x)
x = LeakyReLU()(x)
x = BatchNormalization()(x)
x = Dropout(rate=0.1)(x)
x = Dense(512, kernel_regularizer=tf.keras.regularizers.L2(l2=0.01),
kernel_initializer=tf.keras.initializers.HeNormal(), kernel_constraint=tf.keras.constraints.UnitNorm(axis=0))(x)
x = LeakyReLU()(x)
x = BatchNormalization()(x)
predictions = Dense(num_classes, activation= 'softmax')(x)
model = Model(inputs = base_model.input, outputs = predictions)
Then i compile it because that is necessary at this stage because i have to run the model fit with dummy input before i load the weights. (i think, i have tried to put these code blocks in many different orders to make it work, but i have failed each time)
opt = tfa.optimizers.LazyAdam(lr=0.000074)
model.compile(
loss='sparse_categorical_crossentropy',
optimizer=opt,
metrics=['accuracy']
)
dummy_input = tf.random.uniform([32, 224, 224, 3])
dummy_label = tf.random.uniform([32,])
hist = model.fit(dummy_input, dummy_label)
Then i load the weights for the base model:
base_model.load_weights('/content/drive/MyDrive/MODELS_SAVED/model_RESNET152/model_weights2.h5', by_name=True)
Then i load the weights for the optimizer:
import pickle
with open("/content/drive/MyDrive/weight_values2optimizer.pkl", "rb") as f:
weights = pickle.load(f)
opt = model.optimizer.set_weights(weights)
This results in the following error:
ValueError: You called `set_weights(weights)` on optimizer LazyAdam
with a weight list of length 1245,
but the optimizer was expecting 13 weights.
Provided weights: [63504, array([[[[ 0.00000000e+00, -5.74126025e-04...
Anyone have ideas on how to solve this?
If you have a solution with Adam instead of LazyAdam that is fine too.(i have no idea if that would make a difference)
edit:
I have tried many new things last couple of days but nothing is working. Here is the entire code where i stand right now. It includes both the part where i am saving and the part where i am loading.
import tarfile
my_tar2 = tarfile.open('test.tgz')
my_tar2.extractall('test') # specify which folder to extract to
my_tar2.close()
import zipfile
with zipfile.ZipFile("/content/tot_train_bremoved2.zip", 'r') as zip_ref:
zip_ref.extractall("/content/train/")
import pandas as pd
train_info = pd.read_csv("/content/drive/MyDrive/train_info.csv")
test_info = pd.read_csv("/content/drive/MyDrive/test_info.csv")
train_folder = "/content/train"
test_folder = "/content/test/test"
import tensorflow as tf
import tensorflow.keras as keras
from keras.layers import Input, Lambda, Dense, Flatten, BatchNormalization, Dropout, PReLU, GlobalAveragePooling2D, LeakyReLU, MaxPooling2D
from keras.models import Model
from tensorflow.keras.applications.resnet_v2 import ResNet152V2, preprocess_input
from keras import applications
from keras.preprocessing import image
from keras.preprocessing.image import ImageDataGenerator
from keras.losses import sparse_categorical_crossentropy
from keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping, TensorBoard
import tensorflow_addons as tfa
from sklearn.metrics import confusion_matrix
import numpy as np
import matplotlib.pyplot as plt
num_classes = 423
epochs = 20
batch_size = 32
img_height = 224
img_width = 224
IMAGE_SIZE = [img_height, img_width]
_train_generator = ImageDataGenerator(
rotation_range=180,
zoom_range=0.2,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.3,
horizontal_flip=True,
vertical_flip=True,
preprocessing_function=preprocess_input)
_val_generator = ImageDataGenerator(
preprocessing_function=preprocess_input)
train_generator = _train_generator.flow_from_dataframe(dataframe = train_info,
directory = train_folder, x_col = "filename",
y_col = "artist", seed = 42,
batch_size = batch_size, shuffle = True,
class_mode="sparse", target_size = IMAGE_SIZE)
valid_generator = _val_generator.flow_from_dataframe(dataframe = test_info,
directory = test_folder, x_col = "filename",
y_col = "artist", seed = 42,
batch_size = batch_size, shuffle = True,
class_mode="sparse", target_size = IMAGE_SIZE)
def get_uncompiled_model():
input_tensor = Input(shape=train_generator.image_shape)
base_model = applications.ResNet152(weights='imagenet', include_top=False, input_tensor=input_tensor)
for layer in base_model.layers[:]:
layer.trainable = True
x = Flatten()(base_model.output)
predictions = Dense(num_classes, activation= 'softmax')(x)
model = Model(inputs = base_model.input, outputs = predictions)
return model
opt = keras.optimizers.Adam(lr=0.000074)
def get_compiled_model():
model = get_uncompiled_model()
model.compile(
loss='sparse_categorical_crossentropy',
optimizer=opt,
metrics=['accuracy']
)
return model
earlyStopping = EarlyStopping(monitor='val_loss', patience=5, verbose=0, mode='min')
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=2, verbose=1, min_delta=1e-4, mode='min')
model = get_compiled_model()
from PIL import ImageFile
ImageFile.LOAD_TRUNCATED_IMAGES = True
model.fit(
train_generator,
validation_data=valid_generator,
epochs=epochs,
verbose = 1,
steps_per_epoch=len_train // batch_size,
validation_steps=len_test // batch_size,
callbacks=[earlyStopping, reduce_lr]
)
import keras.backend as K
import pickle
model.save_weights('/content/drive/MyDrive/MODELS_SAVED/model_RESNET152/model_weights5.h5')
symbolic_weights = getattr(model.optimizer, 'weights')
weight_values = K.batch_get_value(symbolic_weights)
with open('/content/drive/MyDrive/MODELS_SAVED/optimizer3.pkl', 'wb') as f:
pickle.dump(weight_values, f)
#Here i am building the new model and its from here i am having problems
input_tensor = Input(shape=train_generator.image_shape)
base_model = applications.ResNet152(weights='imagenet', include_top=False, input_tensor=input_tensor)
for layer in base_model.layers[:]:
layer.trainable = False
x = Flatten()(base_model.output)
x = Dense(512, kernel_regularizer=tf.keras.regularizers.L2(l2=0.01),
kernel_initializer=tf.keras.initializers.HeNormal(),
kernel_constraint=tf.keras.constraints.UnitNorm(axis=0))(x)
x = LeakyReLU()(x)
x = BatchNormalization()(x)
predictions = Dense(num_classes, activation= 'softmax')(x)
model = Model(inputs = base_model.input, outputs = predictions)
model.compile(
loss='sparse_categorical_crossentropy',
optimizer='adam',
metrics=['accuracy']
)
base_model.load_weights('/content/drive/MyDrive/MODELS_SAVED/model_RESNET152/model_weights5.h5', by_name=True)
with open('/content/drive/MyDrive/MODELS_SAVED/optimizer3.pkl', 'rb') as f:
weight_values = pickle.load(f)
model.optimizer.set_weights(weight_values)
from PIL import ImageFile
ImageFile.LOAD_TRUNCATED_IMAGES = True
epochs = 2
model.fit(
train_generator,
validation_data=valid_generator,
epochs=epochs,
steps_per_epoch=len_train // batch_size,
validation_steps=len_test // batch_size,
verbose = 1,
callbacks=[earlyStopping, reduce_lr]
)
Now i am getting the following error running this code block (which above in the complete code is right before the model.fit):
with open('/content/drive/MyDrive/MODELS_SAVED/optimizer3.pkl', 'rb') as f:
weight_values = pickle.load(f)
model.optimizer.set_weights(weight_values)
ValueError: You called `set_weights(weights)` on optimizer Adam with a weight list of length 1245, but the optimizer was expecting 13 weights. Provided weights: [11907, array([[[[ 0.00000000e+00, -8.27514916e-04...
All i am trying to do is to save the weights for the model and optimizer and then build a new model where i am adding a few layers and loading the weights from the base of the model and the weights from the optimizer.

Both models have different architectures so weights of one can't be loaded into another,irrespective of that they inherited same base model. I think it is a simple case of fine-tuning a model (saved model in your case).
What you should do is change the way to create new model, i.e. rather than loading the original resnet model as base model with include_top = False, you should try loading the saved model and implementing your own include_top. This can be done as:
for layer in saved_model.layers[:]:
layer.trainable = False
x = Flatten()(saved_model.layers[-2].output)
Here the key thing is saved_model.layers[-2].output which means output from the second last layer.
Hope it helps, if not please clarify your doubts or let me know what I missed.

Please refer to the following notebook:
https://colab.research.google.com/drive/1j_zLqG1zUMi6UYPdc6gtmkJvHuawL4Sk?usp=sharing
{save,load}_weights on a model includes the weights of the optimizer. That would be the preferred way to initialise the optimizer weights.
You can copy the optimizer from one model to another.
The reason you are getting the error above is that the optimizer doesn't allocate its weights until training starts; If you really want to do it manually, just trigger model.fit() for 1 epoch 1 datapoint and then load the data manually.
You can replace
base_model.load_weights('/content/drive/MyDrive/MODELS_SAVED/model_RESNET152/model_weights5.h5', by_name=True)
with open('/content/drive/MyDrive/MODELS_SAVED/optimizer3.pkl', 'rb') as f:
weight_values = pickle.load(f)
model.optimizer.set_weights(weight_values)
with:
base_model.load_weights('/content/drive/MyDrive/MODELS_SAVED/model_RESNET152/model_weights5.h5', by_name=True)
model.optimizer = base_model.optimizer

After saving the first model's weights with model.save_weights('name.h5'), you should build a second model, exactly like the first one, let's call it model2. Then load the weights you saved before into it. The code should be model2.load_weights('name.h5'). See model.summary() to see the names and number of the first model's layers. For each layer, you need to define a variable and add those weights (and also biases) to that variable with a method called get_weights() . Here is an example:
x1 = model2.layers[1].get_weights()
Here, I put the weights and biases of the first layer (which in mine was a convolution layer) in the variable x1.
x1[0] is a list of the weights of the layer #1.
x1[1] is a list of the biases of the layer #1.

Hyperparameter-Tuning for pretrained NN from TensorFlow

I am starting to learn Python and trying create a neural network, which detects and localizes anomalious parts of images. I am using a pretrained CNN from TensorFlow. The code works so far as it learns my images and classifies them. But when I give him the same amount (around 100) of images for each class. the accuracy is around 50% which is random for two classes. So I am looking for a solution for it right now.
The localization Problem I solving by slicing my images in to multiple parts. The anomaly is being split from non-anomalous region while every image has an reference of its location in its name. So when the image is classified as a certain class one also finds out about its location by its name.
first option: I have to find for a way to extent the amount of my images to look if it improves my accuracy.I didn't work on that yet.
second option: trying to tune the hyper parameters of the CNN, maybe put my images in some earlier layers. I watched a couple tutorials and tried to implement them, but they all failed mostly due to the shape of the arrays or inconsistent numbers.
# Use scikit-learn to grid search the batch size and epochs
import numpy
from sklearn.model_selection import GridSearchCV
from keras.models import Sequential
from keras.layers import Dense
from keras.wrappers.scikit_learn import KerasClassifier
# Function to create model, required for KerasClassifier
def create_model():
# create model
model = Sequential()
model.add(Dense(12, input_dim=8, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
# Compile model
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
return model
# fix random seed for reproducibility
seed = 7
numpy.random.seed(seed)
# load dataset
# create model
model = KerasClassifier(build_fn=create_model, verbose=0)
# define the grid search parameters
batch_size = [10, 20, 40, 60, 80, 100]
epochs = [10, 50, 100]
param_grid = dict(batch_size=batch_size, epochs=epochs)
grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1)
grid_result = grid.fit(x_train, label_batch)
# summarize results
print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
print("%f (%f) with: %r" % (mean, stdev, param))
init = tf.global_variables_initializer()
sess.run(init)
result = model.predict(image_batch)
result.shape
model.compile(
optimizer=tf.train.AdamOptimizer(),
loss='categorical_crossentropy',
metrics=['accuracy'])
class CollectBatchStats(tf.keras.callbacks.Callback):
def __init__(self):
self.batch_losses = []
self.batch_acc = []
def on_batch_end(self, batch, logs=None):
self.batch_losses.append(logs['loss'])
self.batch_acc.append(logs['acc'])
steps_per_epoch = image_data.samples//image_data.batch_size
batch_stats = CollectBatchStats()
model.fit((item for item in image_data), epochs=1,
steps_per_epoch=steps_per_epoch,
callbacks = [batch_stats])

From what you mentioned in the comments you are facing the below error.
ValueError: Error when checking input: expected dense_1_input to have
2 dimensions, but got array with shape (21, 224, 224, 3)
If you are using CNN, why is the First Layer a Dense Layer (I understand it from the name, dense_1_input), instead of a Convolutional Layer.
With the First Layer being the Convolutional Layer, you should pass (224,224,3) for the argument, input_shape
Complete code for Fine Tuning Batch_Size and Number of Epochs for Fashion_MNIST Dataset using CNN is shown below:
# To support both python 2 and python 3
from __future__ import division, print_function, unicode_literals
from io import open
# Common imports
import numpy as np
import os
import tensorflow as tf
from keras.layers import Input, Conv2D, MaxPool2D, Dense, Dropout, Flatten
from keras.models import Sequential
from keras.optimizers import Adam
import matplotlib.pyplot as plt
from keras.regularizers import l1_l2
from matplotlib.pyplot import axis as ax
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.model_selection import GridSearchCV
X = tf.placeholder(tf.float32, shape=[None, 784], name="X")
X_reshaped = tf.reshape(X, shape=[-1, 28, 28, 1])
y = tf.placeholder(tf.int32, shape=[None], name="y")
def create_model():
# instantiate regularizer
Regularizer = l1_l2(0.001)
cnn_model = Sequential()
cnn_model.add(Conv2D(filters = 64,kernel_size = 3, strides=(1, 1), input_shape = (28,28,1),
activation='relu', data_format='channels_last', activity_regularizer=Regularizer))
cnn_model.add(MaxPool2D(pool_size = (2, 2)))
cnn_model.add(Dropout(0.25))
cnn_model.add(Flatten())
cnn_model.add(Dense(units = 32, activation = 'relu', activity_regularizer=Regularizer))
cnn_model.add(Dense(units = 10, activation = 'sigmoid', activity_regularizer=Regularizer))
cnn_model.compile(loss ='sparse_categorical_crossentropy', optimizer=Adam(lr=0.001),metrics =['accuracy'])
return cnn_model
model = KerasClassifier(build_fn=create_model, verbose=0)
(X_train, y_train), (X_test, y_test) = tf.keras.datasets.fashion_mnist.load_data()
X_train = X_train.astype(np.float32).reshape(-1, 28*28) / 255.0
X_train_reshaped = np.reshape(X_train, newshape=[-1, 28, 28, 1])
X_test = X_test.astype(np.float32).reshape(-1, 28*28) / 255.0
X_test_reshaped = np.reshape(X_test, newshape=[-1, 28, 28, 1])
y_train = y_train.astype(int)
y_test = y_test.astype(int)
batch_size = [20, 40]
epochs = [10, 50]
param_grid = dict(batch_size=batch_size, epochs=epochs)
grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=1, cv=3)
grid_result = grid.fit(X_train_reshaped, y_train)
# summarize results
print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
print("%f (%f) with: %r" % (mean, stdev, param))

Tensorflow Embeddings best practice with Estimator

I have a CNN for CIFAR-10 dataset with layers:
[IN] -> [CONV] -> [POOL] -> [CONV] -> [POOL] -> [FC] -> [DROPOUT] -> [LOGITS] -> [OUT]
\-> [EMBEDDINGS]
Estimator code:
config = tf.contrib.learn.RunConfig(save_checkpoints_secs=30)
# Create the Estimator
classifier = tf.estimator.Estimator(model_fn=inference, config=config, model_dir=LOG_DIR)
train_images, train_labels, train_labels_onehot = Utils.load_training_data()
hooks = [
# logging hook
tf.train.LoggingTensorHook(tensors=tensors_to_log, every_n_iter=50),
]
train_input_fn = tf.estimator.inputs.numpy_input_fn(x={'x': train_images}, y=train_labels)
classifier.train(input_fn=train_input_fn, steps=FLAGS.steps, hooks=hooks)
Inference function code:
def inference(self, features, labels, mode):
try:
images = tf.cast(features['x'], tf.float32)
# Input Layer
with tf.name_scope('Data'):
input_layer = tf.reshape(images, [-1, img_width, img_height, num_channels])
# Convolutional Layer 1
with tf.variable_scope('ConvLayer1'):
conv1 = tf.layers.conv2d(inputs=input_layer, filters=32, kernel_size=[5, 5],
padding="same", activation=tf.nn.relu)
pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2)
logging.info('Convolutional Layer 1 build successful..')
# Convolutional Layer 1
with tf.variable_scope('ConvLayer2'):
conv2 = tf.layers.conv2d(inputs=pool1, filters=64, kernel_size=[5, 5],
padding="same", activation=tf.nn.relu)
pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2)
logging.info('Convolutional Layer 2 build successful..')
# Fully Connected Layer
with tf.variable_scope('FullyConnectedLayer'):
pool2_flat = tf.reshape(pool2, [-1, 8 * 8 * 64])
dense = tf.layers.dense(inputs=pool2_flat, units=1024, activation=tf.nn.relu)
dropout = tf.layers.dropout(inputs=dense, rate=0.4,
training=(mode == tf.estimator.ModeKeys.TRAIN))
logging.info('Fully Connected Layer build successful..')
tf.summary.histogram('dropout', dropout)
# Logits Layer
logits = tf.layers.dense(inputs=dropout, units=10)
tf.summary.histogram('logits', logits)
logging.info('Logits Layer build successful..')
predictions = {
# Generate predictions (for PREDICT and EVAL mode)
"classes": tf.argmax(input=logits, axis=1),
# Add `softmax_tensor` to the graph. It is used for PREDICT and by the
# `logging_hook`.
"probabilities": tf.nn.softmax(logits, name="softmax_tensor")
}
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions,
evaluation_hooks=[])
# Calculate Loss (for both TRAIN and EVAL modes)
onehot_labels = tf.one_hot(indices=tf.cast(labels, tf.int32), depth=10)
loss = tf.losses.softmax_cross_entropy(onehot_labels=onehot_labels,
logits=logits)
tf.summary.histogram('loss', loss)
logging.info('Losses build successful..')
# Configure the Training Op (for TRAIN mode)
if mode == tf.estimator.ModeKeys.TRAIN:
learning_rate = tf.train.exponential_decay(start_learning_rate,
tf.train.get_global_step(), 1000, 0.9, staircase=True)
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
train_op = optimizer.minimize(loss=loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op,
scaffold=tf.train.Scaffold(
summary_op=tf.summary.merge_all(),
))
# Add evaluation metrics (for EVAL mode)
accuracy = tf.metrics.accuracy(labels=labels, predictions=predictions["classes"])
tf.summary.histogram('accuracy', accuracy)
logging.info('Accuracy metric build successful..')
return tf.estimator.EstimatorSpec(mode=mode, loss=loss,
train_op=train_op,
scaffold=tf.train.Scaffold(
summary_op=tf.summary.merge_all()
))
I am trying to work with Embeddings Visualization in tensorflow and here I want to visualize dropout output as embeddings.
Code that I've found to use embeddings:
sess = tf.InteractiveSession()
# Input set for Embedded TensorBoard visualization
# Performed with cpu to conserve memory and processing power
with tf.device("/cpu:0"):
embedding = tf.Variable(self._data, trainable=False, name='embedding')
sess.run(embedding.initializer)
writer = tf.summary.FileWriter(LOG_DIR + '/projector', sess.graph)
config = projector.ProjectorConfig()
embed = config.embeddings.add()
embed.tensor_name = embedding.name
embed.metadata_path = os.path.join(LOG_DIR + '/projector/metadata.tsv')
embed.sprite.image_path = os.path.join(DATA_DIR + '/cifar_10k_sprite.png')
embed.sprite.single_image_dim.extend([img_width, img_height])
projector.visualize_embeddings(writer, config)
saver = tf.train.Saver([embedding])
saver.save(sess, os.path.join(LOG_DIR, 'projector/a_model.ckpt'))
It does not work in my case, because I am using Estimator class and I have not access to the session.
Ways that I've tried:
Pass numpy.array variable to the model_fn of Estimator where I can set the value to that variable and then pass that variable to SessionRunHook where I can access session and save data to the file. Did not work because all arguments passed to the Estimator become tensors. So that way does not work cause I'm already have dropout layer tensors.
Create global variable where I can put all values of dropout layer. Did not work too because for that approach I need to access tensors values.
As far as I understood Estimator architecture the main problem is to get dropout layer output signals to the out of the Estimator and pass it somehow to the SessionRunHook to save them as Embeddings. But I think it's not the best way.
What is the right way of using Embeddings in Estimator?

This is how I did it guys (but it may be not the most efficient way):
SessinRunHook:
import tensorflow as tf
from classes.Utils import Utils
class EmbeddingSaverHook(tf.train.SessionRunHook):
def __init__(self, values, labels, captions):
self._saver = None
self._classes = Utils.get_classnames()
self._dense3 = None
self._labels = None
self._emb_values = values
self._emb_labels = labels
self._emb_captions = captions
def begin(self):
self._dense3 = tf.get_default_graph().get_tensor_by_name("dense3/BiasAdd:0")
self._labels = tf.get_default_graph().get_tensor_by_name("labels:0")
def before_run(self, run_context):
return tf.train.SessionRunArgs([self._dense3, self._labels])
def after_run(self, run_context, run_values):
self._emb_values.extend(run_values[0][0])
self._emb_labels.extend(run_values[0][1])
self._emb_captions.extend([self._classes[x] for x in run_values[0][1]])
def end(self, session):
pass
Full code you can see in my github repo

No gradients provided in tensorflow (mean_squared_error)

I'm trying to build a simple net of 2 input neurons (+1 bias) going into 1 output neuron to teach it the "and"-function. It's based on the mnist-clissification example, so it might be overly complex for the task, but it's about the general structure of such nets for me, so please don't say "you can just do it in numpy" or something, it's about tensorflow NNs for me. So here is the code:
import tensorflow as tf
import numpy as np
tf.logging.set_verbosity(tf.logging.INFO)
def model_fn(features, labels, mode):
input_layer = tf.reshape(features["x"], [-1, 2])
output_layer = tf.layers.dense(inputs=input_layer, units=1, activation=tf.nn.relu, name="output_layer")
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode, predictions=output_layer)
loss = tf.losses.mean_squared_error(labels=labels, predictions=output_layer)
if mode == tf.estimator.ModeKeys.TRAIN:
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)
train_op = optimizer.minimize(loss=loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op)
eval_metrics_ops = {"accuracy": tf.metrics.accuracy(labels=labels, predictions=output_layer)}
return tf.estimator.EstimatorSpec(mode=mode, predictions=output_layer, loss=loss)
def main(unused_arg):
train_data = np.asarray(np.reshape([[0,0],[0,1],[1,0],[1,1]],[4,2]))
train_labels = np.asarray(np.reshape([0,0,0,1],[4,1]))
eval_data = train_data
eval_labels = train_labels
classifier = tf.estimator.Estimator(model_fn=model_fn, model_dir="/tmp/NN_AND")
tensors_to_log = {"The output:": "output_layer"}
logging_hook = tf.train.LoggingTensorHook(tensors=tensors_to_log,every_n_iter=10)
train_input_fn = tf.estimator.inputs.numpy_input_fn(x={"x":train_data}, y=train_labels, batch_size=10, num_epochs=None, shuffle=True)
classifier.train(input_fn=train_input_fn, steps=2000, hooks=[logging_hook])
eval_input_fn = tf.estimator.inputs.numpy_input_fn(x={"x":eval_data}, y=eval_labels, batch_size=1, shuffle=False)
eval_results = classifier.evaluate(input_fn=eval_input_fn)
print(eval_results)
if __name__ == "__main__":
tf.app.run()

I've made few slight modifications to your code which enable learning the and function:
1) change your train_data to float32 representation.
train_data = np.asarray(np.reshape([[0,0],[0,1],[1,0],[1,1]],[4,2]), dtype=np.float32)`
2) Remove relu activation from the output layer - generally speaking, using relus in the output layer is not recommended. This might lead to dead relus and all gradients will be equal zero, which in turn will not make any learning possible.
output_layer = tf.layers.dense(inputs=input_layer, units=1, activation=None, name="output_layer")
3) In your eval_metrics_ops make sure you round the result so you can actually measure accuracy:
eval_metrics_ops = {"accuracy": tf.metrics.accuracy(labels=labels, predictions=tf.round(output_layer))}
4) Don't fortget to add the eval_metrics_ops parameter you defined to the estimator:
return tf.estimator.EstimatorSpec(mode=mode, predictions=output_layer, loss=loss, eval_metric_ops=eval_metrics_ops)
In addition, to log the last layer output you should use:
tensors_to_log = {"The output:": "output_layer/BiasAdd:0"}