I am developing a Keras model for the multi-class classification problem (4 classes) with a custom metric.
The problem is that I cannot develop a custom metric for this model. When I run the model, the values of metrics are empty.
This is my model:
nb_classes = 4
model = Sequential()
model.add(LSTM(
units=50,
return_sequences=True,
input_shape=(20,18),
dropout=0.2,
recurrent_dropout=0.2
)
)
model.add(Dropout(0.2))
model.add(Flatten())
model.add(Dense(units=nb_classes,
activation='softmax'))
model.compile(loss="categorical_crossentropy",optimizer='adadelta')
history = model.fit(np.array(X_train), y_train,
validation_data=(np.array(X_test), y_test),
epochs=50,
batch_size=2,
callbacks=[model_metrics],
shuffle=False,
verbose=1)
This is how model_metrics is defined:
class Metrics(Callback):
def on_train_begin(self, logs={}):
self.val_f1s = []
self.val_recalls = []
self.val_precisions = []
def on_epoch_end(self, epoch, logs={}):
val_predict = np.argmax((np.asarray(self.model.predict(self.validation_data[0]))).round(), axis=1)
val_targ = np.argmax(self.validation_data[1], axis=1)
_val_f1 = metrics.f1_score(val_targ, val_predict, average='weighted')
_val_recall = metrics.recall_score(val_targ, val_predict, average='weighted')
_val_precision = metrics.precision_score(val_targ, val_predict, average='weighted')
self.val_f1s.append(_val_f1)
self.val_recalls.append(_val_recall)
self.val_precisions.append(_val_precision)
print(" — val_f1: %f — val_precision: %f — val_recall %f".format(_val_f1, _val_precision, _val_recall))
return
model_metrics = Metrics()
When I run fit, I get this result:
Train on 400 samples, validate on 80 samples
Epoch 1/50
400/400 [==============================] - 7s 17ms/step - loss: 0.6892 - val_loss: 4.8016
— val_f1: %f — val_precision: %f — val_recall %f
Epoch 2/50
20/400 [>.............................] - ETA: 3s - loss: 2.8010
/Users/tau/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
/Users/tau/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples.
'precision', 'predicted', average, warn_for)
400/400 [==============================] - 3s 9ms/step - loss: 0.7593 - val_loss: 4.5832
— val_f1: %f — val_precision: %f — val_recall %f
Epoch 3/50
400/400 [==============================] - 4s 9ms/step - loss: 0.6809 - val_loss: 4.9039
— val_f1: %f — val_precision: %f — val_recall %f
You can see val_f1: %f — val_precision: %f — val_recall %f. There are no values of metrics. Why? What am I doing wrong?
Your problem is not in Keras. You are using the Python string formatting wrong. Here is the correct usage:
print(" — val_f1: {:f} — val_precision: {:f} — val_recall {:f}".format(_val_f1, _val_precision, _val_recall))
Alternatively:
print(" — val_f1: %f — val_precision: %f — val_recall %f" % (_val_f1, _val_precision, _val_recall))
Related
I'm trying to create a small transformer model with Keras to model stock prices, based off of this tutorial from the Keras docs. The problem is, my test loss is massive and barely changes between epochs, unsurprisingly resulting in severe underfitting, with my outputs all the same arbitrary value.
My code is below:
def transformer_encoder_block(inputs, head_size, num_heads, filters, dropout=0):
# Normalization and Attention
x = layers.LayerNormalization(epsilon=1e-6)(inputs)
x = layers.MultiHeadAttention(
key_dim=head_size, num_heads=num_heads, dropout=dropout
)(x, x)
x = layers.Dropout(dropout)(x)
res = x + inputs
# Feed Forward Part
x = layers.LayerNormalization(epsilon=1e-6)(res)
x = layers.Conv1D(filters=filters, kernel_size=1, activation="relu")(x)
x = layers.Dropout(dropout)(x)
x = layers.Conv1D(filters=inputs.shape[-1], kernel_size=1)(x)
return x + res
data = ...
input = np.array(
keras.preprocessing.sequence.pad_sequences(data["input"], padding="pre", dtype="float32"))
output = np.array(
keras.preprocessing.sequence.pad_sequences(data["output"], padding="pre", dtype="float32"))
# Input shape: (723, 36, 22)
# Output shape: (723, 36, 1)
# Train data
train_features = input[100:]
train_labels = output[100:]
train_labels = tf.keras.utils.to_categorical(train_labels, num_classes=3)
# Test data
test_features = input[:100]
test_labels = output[:100]
test_labels = tf.keras.utils.to_categorical(test_labels, num_classes=3)
inputs = keras.Input(shape=(None,22), dtype="float32", name="inputs")
# Ignore padding in inputs
x = layers.Masking(mask_value=0)(inputs)
x = transformer_encoder_block(x, head_size=64, num_heads=16, filters=3, dropout=0.2)
# Multiclass = Softmax (decrease, no change, increase)
outputs = layers.TimeDistributed(layers.Dense(3, activation="softmax", name="outputs"))(x)
# Create model
model = keras.Model(inputs=inputs, outputs=outputs)
# Compile model
model.compile(loss="categorical_crossentropy", optimizer=(tf.keras.optimizers.Adam(learning_rate=0.005)), metrics=['accuracy'])
# Train model
history = model.fit(train_features, train_labels, epochs=10, batch_size=32)
# Evaluate on the test data
test_loss = model.evaluate(test_features, test_labels, verbose=0)
print("Test loss:", test_loss)
out = model.predict(test_features)
After padding, input is of shape (723, 36, 22), and output is of shape (723, 36, 1) (before converting output to one hop, after which there are 3 output classes).
Here's an example output for ten epochs (trust me, more than ten doesn't make it better):
Epoch 1/10
20/20 [==============================] - 2s 62ms/step - loss: 10.7436 - accuracy: 0.3335
Epoch 2/10
20/20 [==============================] - 1s 62ms/step - loss: 10.7083 - accuracy: 0.3354
Epoch 3/10
20/20 [==============================] - 1s 60ms/step - loss: 10.6555 - accuracy: 0.3392
Epoch 4/10
20/20 [==============================] - 1s 62ms/step - loss: 10.7846 - accuracy: 0.3306
Epoch 5/10
20/20 [==============================] - 1s 60ms/step - loss: 10.7600 - accuracy: 0.3322
Epoch 6/10
20/20 [==============================] - 1s 59ms/step - loss: 10.7074 - accuracy: 0.3358
Epoch 7/10
20/20 [==============================] - 1s 59ms/step - loss: 10.6569 - accuracy: 0.3385
Epoch 8/10
20/20 [==============================] - 1s 60ms/step - loss: 10.7767 - accuracy: 0.3314
Epoch 9/10
20/20 [==============================] - 1s 61ms/step - loss: 10.7346 - accuracy: 0.3341
Epoch 10/10
20/20 [==============================] - 1s 62ms/step - loss: 10.7093 - accuracy: 0.3354
Test loss: [10.073813438415527, 0.375]
4/4 [==============================] - 0s 22ms/step
Using the same data on a simple LSTM model with the same shape yielded a desirable prediction with a constantly decreasing loss.
Tweaking the learning rate appears to have no effect, nor does stacking more transformer_encoder_block()s.
If anyone has any suggestions for how I can solve this, please let me know.
I have an existing setup using scikit-learn, but am looking into expanding into deep learning with Keras. I am also using Dask, which recommends using SciKeras.
The way the SciKeras KerasClassifier is currently setup, seems to fit as expected (from the verbose output), but the model seems to have learned nothing at all. I have followed the SciKeras docs here, but I might have overlooked something.
With a Scikit-Learn RF Classifier the kappa score is about 0.44, with
Keras it is about 0.55, and with SciKeras it is 0.0 (clearly an
issue). In the 2. Following SciKeras docs to use Keras where
is the implementation error that prevents a similar result compared to
the one achieved using the 3. Exclusively using Keras below?
Below I have listed the current scikit-learn implementation with RF (as expected output), the output with SciKeras (as actual output), and the output using Keras exclusively (as expected result)
1. Current output using scikit-learn random forest:
def default_classifier():
return RandomForestClassifier(oob_score=True, n_jobs=-1)
... ### Preprocessing stuff...
X_train, X_test, y_train, y_test = splits
# Define the Pipeline
## Classification
model = default_classifier()
model.fit(X_train, y_train)
## Evaluation Metrics
from sklearn.model_selection import cross_val_score
score = cross_val_score(model, X_test, y_test, scoring='accuracy', cv=5, n_jobs=-1, error_score='raise')
print('Mean: %.3f (Std: %.3f)' % (np.mean(score), np.std(score)))
# Verbose with results...
columns, report, true_matrix, pred_matrix = cl.classification_metrics(model, splits, score)
Respective sklearn output:
Test Size: 0.2
Split Shapes: [(79997, 96), (20000, 96), (79997, 12), (20000, 12)]
Mean: 0.374 (Std: 0.006)
Overall: 0.510 Kappa: 0.441
Weighted F1-Score: 0.539
2. Following SciKeras docs to use Keras:
from tensorflow import keras
from scikeras.wrappers import KerasClassifier
from sklearn.model_selection import train_test_split
import numpy as np
def fcn_model(hidden_layer_dim, meta):
# note that meta is a special argument that will be
# handed a dict containing input metadata
n_features_in_ = meta["n_features_in_"]
X_shape_ = meta["X_shape_"]
n_classes_ = meta["n_classes_"]
model = keras.models.Sequential()
model.add(keras.layers.Dense(n_features_in_, input_shape=X_shape_[1:]))
model.add(keras.layers.Activation("relu"))
model.add(keras.layers.Dense(hidden_layer_dim))
model.add(keras.layers.Activation("relu"))
model.add(keras.layers.Dense(n_classes_))
model.add(keras.layers.Activation("softmax"))
return model
def get_model_fcn(modelargs={}):
return KerasClassifier(fcn_model,
hidden_layer_dim=128,
epochs=10,
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'],
fit__use_multiprocessing=True,
**modelargs)
... ### Preprocessing stuff...
X_train, X_test, y_train, y_test = splits
# Define the Pipeline
## Classification
model = get_model_fcn()
model.fit(X_train, y_train)
## Evaluation Metrics
from sklearn.model_selection import cross_val_score
score = cross_val_score(model, X_test, y_test, scoring='accuracy', cv=5, n_jobs=-1, error_score='raise')
print('Mean: %.3f (Std: %.3f)' % (np.mean(score), np.std(score)))
columns, report, true_matrix, pred_matrix = cl.classification_metrics(model, splits, score)
Respective scikeras output (result not very good):
Test Size: 0.2
Split Shapes: [(79997, 96), (20000, 96), (79997, 12), (20000, 12)]
Epoch 1/10
2500/2500 [==============================] - 4s 1ms/step - loss: 1.6750 - accuracy: 0.3762
Epoch 2/10
2500/2500 [==============================] - 3s 1ms/step - loss: 1.3132 - accuracy: 0.5021
Epoch 3/10
2500/2500 [==============================] - 3s 1ms/step - loss: 1.2295 - accuracy: 0.5371
Epoch 4/10
2500/2500 [==============================] - 3s 1ms/step - loss: 1.1651 - accuracy: 0.5599
Epoch 5/10
2500/2500 [==============================] - 3s 1ms/step - loss: 1.1178 - accuracy: 0.5806
Epoch 6/10
2500/2500 [==============================] - 3s 1ms/step - loss: 1.0889 - accuracy: 0.5935
Epoch 7/10
2500/2500 [==============================] - 3s 1ms/step - loss: 1.0845 - accuracy: 0.5922
Epoch 8/10
2500/2500 [==============================] - 3s 1ms/step - loss: 1.0548 - accuracy: 0.6043
Epoch 9/10
2500/2500 [==============================] - 3s 1ms/step - loss: 1.0415 - accuracy: 0.6117
Epoch 10/10
2500/2500 [==============================] - 3s 1ms/step - loss: 1.0316 - accuracy: 0.6172
Mean: 0.000 (Std: 0.000)
625/625 [==============================] - 0s 700us/step # Here it is running model.predict(X_test)
Overall: 0.130 Kappa: 0.000
Weighted F1-Score: 0.030
3. Exclusively using Keras:
# meta copies what SciKeras passes to the Keras model
meta = {
#'classes_': array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]),
#'target_type_': 'multilabel-indicator',
'y_dtype_': np.dtype('uint8'),
'y_ndim_': 2,
'X_dtype_': np.dtype('float32'),
'X_shape_': (79997, 96),
'n_features_in_': 96,
#'target_encoder_': ClassifierLabelEncoder(loss='categorical_crossentropy'),
'n_classes_': 12,
'n_outputs_': 1,
'n_outputs_expected_': 1,
#'feature_encoder_': FunctionTransformer()
}
def fcn_model(hidden_layer_dim, meta):
# note that meta is a special argument that will be
# handed a dict containing input metadata
n_features_in_ = meta["n_features_in_"]
X_shape_ = meta["X_shape_"]
n_classes_ = meta["n_classes_"]
model = keras.models.Sequential()
model.add(keras.layers.Dense(n_features_in_, input_shape=X_shape_[1:]))
model.add(keras.layers.Activation("relu"))
model.add(keras.layers.Dense(hidden_layer_dim))
model.add(keras.layers.Activation("relu"))
model.add(keras.layers.Dense(n_classes_))
model.add(keras.layers.Activation("softmax"))
return model
def get_model_fcn(modelargs={}):
model = fcn_model(128, meta)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
... ### Preprocessing stuff...
X_train, X_test, y_train, y_test = splits
# Define the Pipeline
## Classification
model = get_model_fcn()
model.fit(X_train, y_train, epochs=10)
## Evaluation Metrics
#from sklearn.model_selection import cross_val_score
#score = cross_val_score(model, X_test, y_test, scoring='accuracy', cv=5, n_jobs=-1, #error_score='raise')
#print('Mean: %.3f (Std: %.3f)' % (np.mean(score), np.std(score)))
columns, report, true_matrix, pred_matrix = cl.classification_metrics(model, splits, score)
Expected output from using Keras:
Test Size: 0.2
Split Shapes: [(79997, 96), (20000, 96), (79997, 12), (20000, 12)]
Epoch 1/10
2500/2500 [==============================] - 3s 1ms/step - loss: 1.6941 - accuracy: 0.3730
Epoch 2/10
2500/2500 [==============================] - 3s 1ms/step - loss: 1.3193 - accuracy: 0.5002
Epoch 3/10
2500/2500 [==============================] - 3s 1ms/step - loss: 1.2206 - accuracy: 0.5399
Epoch 4/10
2500/2500 [==============================] - 3s 1ms/step - loss: 1.1585 - accuracy: 0.5613
Epoch 5/10
2500/2500 [==============================] - 3s 1ms/step - loss: 1.1221 - accuracy: 0.5758
Epoch 6/10
2500/2500 [==============================] - 3s 1ms/step - loss: 1.0923 - accuracy: 0.5928
Epoch 7/10
2500/2500 [==============================] - 3s 1ms/step - loss: 1.0682 - accuracy: 0.5984
Epoch 8/10
2500/2500 [==============================] - 3s 1ms/step - loss: 1.0611 - accuracy: 0.6046
Epoch 9/10
2500/2500 [==============================] - 3s 1ms/step - loss: 1.0445 - accuracy: 0.6138
Epoch 10/10
2500/2500 [==============================] - 3s 1ms/step - loss: 1.0236 - accuracy: 0.6186
Overall: 0.601 Kappa: 0.548
Weighted F1-Score: 0.600
Apparently it was a bug with how it handled multi-class one-hot encoded targets, issue handled here
I am trying to build an image classifier that differentiates images into pumps, Turbines, and PCB classes. I am using transfer learning from Inception V3.
Below is my code to initialize InceptionV3
import os
from tensorflow.keras import layers
from tensorflow.keras import Model
!wget --no-check-certificate \
https://storage.googleapis.com/mledu-datasets/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5 \
-O /tmp/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5
from tensorflow.keras.applications.inception_v3 import InceptionV3
local_weights_file = '/tmp/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5'
pre_trained_model = InceptionV3(input_shape = (150, 150, 3),
include_top = False,
weights = None)
pre_trained_model.load_weights(local_weights_file)
for layer in pre_trained_model.layers:
layer.trainable = False
# pre_trained_model.summary()
last_layer = pre_trained_model.get_layer('mixed7')
print('last layer output shape: ', last_layer.output_shape)
last_output = last_layer.output
Next I connect my DNN to the pre-trained model:
from tensorflow.keras.optimizers import RMSprop
# Flatten the output layer to 1 dimension
x = layers.Flatten()(last_output)
# Add a fully connected layer with 1,024 hidden units and ReLU activation
x = layers.Dense(1024, activation='relu')(x)
# Add a dropout rate of 0.2
x = layers.Dropout(0.2)(x)
x = layers.Dense (3, activation='softmax')(x)
model = Model( pre_trained_model.input, x)
model.compile(optimizer = RMSprop(lr=0.0001),
loss = 'categorical_crossentropy',
metrics = ['accuracy'])
I feed in my images using ImageDataGenerator and train the model as below:
history = model.fit(
train_generator,
validation_data = validation_generator,
steps_per_epoch = 100,
epochs = 20,
validation_steps = 50,
verbose = 2)
However, the validation accuracy is not printed/generated after the first epoch:
Epoch 1/20
/usr/local/lib/python3.6/dist-packages/PIL/TiffImagePlugin.py:788: UserWarning: Corrupt EXIF data. Expecting to read 4 bytes but only got 0.
warnings.warn(str(msg))
/usr/local/lib/python3.6/dist-packages/PIL/Image.py:932: UserWarning: Palette images with Transparency expressed in bytes should be converted to RGBA images
"Palette images with Transparency expressed in bytes should be "
WARNING:tensorflow:Your input ran out of data; interrupting training. Make sure that your dataset or generator can generate at least `steps_per_epoch * epochs` batches (in this case, 50 batches). You may need to use the repeat() function when building your dataset.
100/100 - 43s - loss: 0.1186 - accuracy: 0.9620 - val_loss: 11.7513 - val_accuracy: 0.3267
Epoch 2/20
100/100 - 41s - loss: 0.1299 - accuracy: 0.9630
Epoch 3/20
100/100 - 39s - loss: 0.0688 - accuracy: 0.9840
Epoch 4/20
100/100 - 39s - loss: 0.0826 - accuracy: 0.9785
Epoch 5/20
100/100 - 39s - loss: 0.0909 - accuracy: 0.9810
Epoch 6/20
100/100 - 39s - loss: 0.0523 - accuracy: 0.9845
Epoch 7/20
100/100 - 38s - loss: 0.0976 - accuracy: 0.9835
Epoch 8/20
100/100 - 39s - loss: 0.0802 - accuracy: 0.9795
Epoch 9/20
100/100 - 39s - loss: 0.0612 - accuracy: 0.9860
Epoch 10/20
100/100 - 40s - loss: 0.0729 - accuracy: 0.9825
Epoch 11/20
100/100 - 39s - loss: 0.0601 - accuracy: 0.9870
Epoch 12/20
100/100 - 39s - loss: 0.0976 - accuracy: 0.9840
Epoch 13/20
100/100 - 39s - loss: 0.0591 - accuracy: 0.9815
Epoch 14/20
I am not understanding as to what is stopping the validation accuracy from being printed/generated. I get an error if the plot a graph on accuracy vs validation accuracy with a message as:
ValueError: x and y must have same first dimension, but have shapes (20,) and (1,)
what am I missing here?
It worked finally, posting my changes here in case if anybody faces issues like these.
So I changed the "weights" parameter in InceptionV3 from None to 'imagenet' and calculated my steps per epoch and validations steps as follows:
steps_per_epoch = np.ceil(no_of_training_images/batch_size)
validation_steps = np.ceil(no_of validation_images/batch_size)
As you see WARNING:tensorflow:Your input ran out of data; interrupting training. Make sure that your dataset or generator can generate at least ``steps_per_epoch * epochs`` batches (in this case, 50 batches). You may need to use the repeat() function when building your dataset.
To make sure that you have "at least steps_per_epoch * epochs batches", set the steps_per_epoch to:
steps_per_epoch = X_train.shape[0]//batch_size
I have an unbalanced dataset (only 0.06% of data is labeled 1 and the rest are labeled 0). As I researched, I had to resample the data, so I used imblearn package to randomUnserSample my dataset. Then I used a Keras Sequential to create a neural network. While training, F1Score increases to around 75% (1000th epoch result is: loss: 0.5691 - acc: 0.7543 - f1_m: 0.7525 - precision_m: 0.7582 - recall_m: 0.7472), but on test set, the result is disappointing (loss: 55.35181%, acc: 79.25248%, f1_m: 0.39789%, precision_m: 0.23259%, recall_m: 1.54982%).
What I assume is that on train set, because the number of 1s and 0s are the same and therefore class_wights are both set to 1, so the network is not costing much for wrong predictions of 1s.
I have used some techniques like reducing the number of layers, reducing number of neurons, using regularization and dropouts, but the test set f1Score is never more than 0.5%. What should I do. Thanks
my neural network:
def neural_network(X, y, epochs_count=3, handle_overfit=False):
# create model
model = Sequential()
model.add(Dense(12, input_dim=len(X_test.columns), activation='relu'))
if (handle_overfit):
model.add(Dropout(rate = 0.5))
model.add(Dense(8, activation='relu', kernel_regularizer=regularizers.l1(0.1)))
if (handle_overfit):
model.add(Dropout(rate = 0.1))
model.add(Dense(1, activation='sigmoid'))
# compile the model
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['acc', f1_m, precision_m, recall_m])
# change weights of the classes '0' and '1' and set weights automatically
class_weights = class_weight.compute_class_weight('balanced', [0, 1], y)
print("---------------------- \n chosen class_wieghts are: ", class_weights, " \n ---------------------")
# Fit the model
model.fit(X, y, epochs=epochs_count, batch_size=512, class_weight=class_weights)
return model
defining train and test set:
vtrain_set, test_set = train_test_split(data, test_size=0.35, random_state=0)
X_train = train_set[['..... some columns ....']]
y_train = train_set[['success']]
print('Initial dataset shape: ', X_train.shape)
rus = RandomUnderSampler(random_state=42)
X_undersampled, y_undersampled = rus.fit_sample(X_train, y_train)
print('undersampled dataset shape: ', X_undersampled.shape)
and result is:
Initial dataset shape: (1625843, 11)
undersampled dataset shape: (1970, 11)
and finally the neural network calling:
print (X_undersampled.shape, y_undersampled.shape)
print (X_test.shape, y_test.shape)
model = neural_network(X_undersampled, y_undersampled, 1000, handle_overfit=True)
# evaluate the model
print("\n---------------\nEvaluated on test set:")
scores = model.evaluate(X_test, y_test)
for i in range(len(model.metrics_names)):
print("%s: %.5f%%" % (model.metrics_names[i], scores[i]*100))
and result is:
(1970, 11) (1970,)
(875454, 11) (875454, 1)
----------------------
chosen class_wieghts are: [1. 1.]
---------------------
Epoch 1/1000
1970/1970 [==============================] - 4s 2ms/step - loss: 4.5034 - acc: 0.5147 - f1_m: 0.3703 - precision_m: 0.5291 - recall_m: 0.2859
.
.
.
.
Epoch 999/1000
1970/1970 [==============================] - 0s 6us/step - loss: 0.5705 - acc: 0.7538 - f1_m: 0.7471 - precision_m: 0.7668 - recall_m: 0.7296
Epoch 1000/1000
1970/1970 [==============================] - 0s 6us/step - loss: 0.5691 - acc: 0.7543 - f1_m: 0.7525 - precision_m: 0.7582 - recall_m: 0.7472
---------------
Evaluated on test set:
875454/875454 [==============================] - 49s 56us/step
loss: 55.35181%
acc: 79.25248%
f1_m: 0.39789%
precision_m: 0.23259%
recall_m: 1.54982%
I'd like to monitor eg. the learning rate during training in Keras both in the progress bar and in Tensorboard. I figure there must be a way to specify which variables are logged, but there's no immediate clarification on this issue on the Keras website.
I guess it's got something to do with creating a custom Callback function, however, it should be possible to modify the already existing progress bar callback, no?
It can be achieved via a custom metric. Take the learning rate as an example:
def get_lr_metric(optimizer):
def lr(y_true, y_pred):
return optimizer.lr
return lr
x = Input((50,))
out = Dense(1, activation='sigmoid')(x)
model = Model(x, out)
optimizer = Adam(lr=0.001)
lr_metric = get_lr_metric(optimizer)
model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['acc', lr_metric])
# reducing the learning rate by half every 2 epochs
cbks = [LearningRateScheduler(lambda epoch: 0.001 * 0.5 ** (epoch // 2)),
TensorBoard(write_graph=False)]
X = np.random.rand(1000, 50)
Y = np.random.randint(2, size=1000)
model.fit(X, Y, epochs=10, callbacks=cbks)
The LR will be printed in the progress bar:
Epoch 1/10
1000/1000 [==============================] - 0s 103us/step - loss: 0.8228 - acc: 0.4960 - lr: 0.0010
Epoch 2/10
1000/1000 [==============================] - 0s 61us/step - loss: 0.7305 - acc: 0.4970 - lr: 0.0010
Epoch 3/10
1000/1000 [==============================] - 0s 62us/step - loss: 0.7145 - acc: 0.4730 - lr: 5.0000e-04
Epoch 4/10
1000/1000 [==============================] - 0s 58us/step - loss: 0.7129 - acc: 0.4800 - lr: 5.0000e-04
Epoch 5/10
1000/1000 [==============================] - 0s 58us/step - loss: 0.7124 - acc: 0.4810 - lr: 2.5000e-04
Epoch 6/10
1000/1000 [==============================] - 0s 63us/step - loss: 0.7123 - acc: 0.4790 - lr: 2.5000e-04
Epoch 7/10
1000/1000 [==============================] - 0s 61us/step - loss: 0.7119 - acc: 0.4840 - lr: 1.2500e-04
Epoch 8/10
1000/1000 [==============================] - 0s 61us/step - loss: 0.7117 - acc: 0.4880 - lr: 1.2500e-04
Epoch 9/10
1000/1000 [==============================] - 0s 59us/step - loss: 0.7116 - acc: 0.4880 - lr: 6.2500e-05
Epoch 10/10
1000/1000 [==============================] - 0s 63us/step - loss: 0.7115 - acc: 0.4880 - lr: 6.2500e-05
Then, you can visualize the LR curve in TensorBoard.
Another way (in fact encouraged one) of how to pass custom values to TensorBoard is by sublcassing the keras.callbacks.TensorBoard class. This allows you to apply custom functions to obtain desired metrics and pass them directly to TensorBoard.
Here is an example for learning rate of Adam optimizer:
class SubTensorBoard(TensorBoard):
def __init__(self, *args, **kwargs):
super(SubTensorBoard, self).__init__(*args, **kwargs)
def lr_getter(self):
# Get vals
decay = self.model.optimizer.decay
lr = self.model.optimizer.lr
iters = self.model.optimizer.iterations # only this should not be const
beta_1 = self.model.optimizer.beta_1
beta_2 = self.model.optimizer.beta_2
# calculate
lr = lr * (1. / (1. + decay * K.cast(iters, K.dtype(decay))))
t = K.cast(iters, K.floatx()) + 1
lr_t = lr * (K.sqrt(1. - K.pow(beta_2, t)) / (1. - K.pow(beta_1, t)))
return np.float32(K.eval(lr_t))
def on_epoch_end(self, episode, logs = {}):
logs.update({"lr": self.lr_getter()})
super(SubTensorBoard, self).on_epoch_end(episode, logs)
I've come to this question because I wanted to log more variables in the Keras progress bar. This is the way I did it after reading the answers here:
class UpdateMetricsCallback(tf.keras.callbacks.Callback):
def on_batch_end(self, batch, logs):
logs.update({'my_batch_metric' : 0.1, 'my_other_batch_metric': 0.2})
def on_epoch_end(self, epoch, logs):
logs.update({'my_epoch_metric' : 0.1, 'my_other_epoch_metric': 0.2})
model.fit(...,
callbacks=[UpdateMetricsCallback()]
)
I hope it helps others.