We Keep Coding

How to print infomation in the get_gradient function?

I want to change the get_gradients method of Adam to execute gradient Centralization, the following code is from https://keras.io/examples/vision/gradient_centralization/
My question is why the code print('ttt') in method get_gradients prints nothing when fit function is executing?
print('dddd') works and print dddd in screen.
from tensorflow.keras.optimizers import Adam
import tensorflow as tf
class GCAdam(Adam):
def __init__(self, learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-7, amsgrad=False, name='Adam', **kwargs):
super().__init__(learning_rate=learning_rate, beta_1=beta_1, beta_2=beta_2, epsilon=epsilon, amsgrad=amsgrad, name=name, **kwargs)
print('dddd')
def get_gradients(self, loss, params):
# We here just provide a modified get_gradients() function since we are
# trying to just compute the centralized gradients.
grads = []
gradients = super().get_gradients()
print('ttt')
for grad in gradients:
grad_len = len(grad.shape)
if grad_len > 1:
axis = list(range(grad_len - 1))
grad -= tf.reduce_mean(grad, axis=axis, keep_dims=True)
grads.append(grad)
return grads
optimizergc = GCAdam(learning_rate=1e-4)
model.compile(loss="categorical_crossentropy", optimizer=optimizergc, metrics=["accuracy"])
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_split=0.1,verbose=1)

Issue with custom metric auc callback for keras

I am trying to implement custom roccallback for keras. The call back function I wrote is below. I have to achieve a target of 0.90.
AUROC callback:
class rocback(Callback):
def __init__(self, validation_data):
super(rocback, self).__init__()
# self.training_data = training_data
self.validation_data = validation_data
def on_train_begin(self, logs={}) :
return
def on_epoch_end(self, epoch, logs={}):
probs = self.model.predict(self.validation_data[0])
probs = np.round(probs)
y_true = self.validation_data[1]
y_true = np.round(y_true)
score = roc_auc_score(y_true, probs, average='micro')
logs['auc'] = score
Hence next callback which I wrote was to achieve the target.
class scoreTarget(Callback):
def __init__(self, target):
super(scoreTarget, self).__init__()
self.target = target
def on_epoch_end(self, epoch, logs={}):
acc = logs['auc']
if acc >= self.target:
self.model.stop_training = True
The list of call back used is below:
roc_callback = rocback((X_test_pooled_output, y_test))
early_stopping = EarlyStopping(patience=5)
tensorboard = TensorBoard()
reduce_lr = ReduceLROnPlateau(patience=3)
target = scoreTarget(0.90)
callbacks = [
roc_callback,
early_stopping,
tensorboard,
reduce_lr,
target,
]
The classifier function which I wrote is below:
class ReviewClassifier(Model):
def __init__(self):
super(ReviewClassifier, self).__init__()
self.dense_1 = Dense(64, activation='relu')
self.dense_2 = Dense(32, activation='relu')
self.dense_3 = Dense(16, activation='relu')
self.classify = Dense(1, activation='sigmoid')
self.dropout_1 = Dropout(0.2)
self.dropout_2 = Dropout(0.2)
self.dropout_3 = Dropout(0.2)
def call(self, inputs):
x = self.dense_1(inputs)
x = self.dropout_1(x)
x = self.dense_2(x)
x = self.dropout_2(x)
x = self.dense_3(x)
x = self.dropout_3(x)
x = self.classify(x)
return x
review_classifier = ReviewClassifier()
review_classifier.build((None, 768))
review_classifier.summary()
The compile funtcion I wrote is this:
review_classifier.compile(loss='binary_crossentropy',
optimizer='adam',metrics=[rocback])
fit function:
!rm -rf ./logs/*
history = review_classifier.fit(X_train_pooled_output, y_train,
batch_size=32, epochs=100,
callbacks=callbacks,
validation_data=(X_test_pooled_output, y_test))
The error received is:
Epoch 1/100
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-54-fd7595a2c88c> in <module>()
1 get_ipython().system('rm -rf ./logs/*')
----> 2 history = review_classifier.fit(X_train_pooled_output, y_train, batch_size=32, epochs=100,callbacks=callbacks, validation_data=(X_test_pooled_output, y_test))
9 frames
/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/func_graph.py in wrapper(*args, **kwargs)
975 except Exception as e: # pylint:disable=broad-except
976 if hasattr(e, "ag_error_metadata"):
--> 977 raise e.ag_error_metadata.to_exception(e)
978 else:
979 raise
TypeError: in user code:
/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py:805 train_function *
return step_function(self, iterator)
/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py:795 step_function **
outputs = model.distribute_strategy.run(run_step, args=(data,))
/usr/local/lib/python3.7/dist-packages/tensorflow/python/distribute/distribute_lib.py:1259 run
return self._extended.call_for_each_replica(fn, args=args, kwargs=kwargs)
/usr/local/lib/python3.7/dist-packages/tensorflow/python/distribute/distribute_lib.py:2730 call_for_each_replica
return self._call_for_each_replica(fn, args, kwargs)
/usr/local/lib/python3.7/dist-packages/tensorflow/python/distribute/distribute_lib.py:3417 _call_for_each_replica
return fn(*args, **kwargs)
/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py:788 run_step **
outputs = model.train_step(data)
/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py:758 train_step
self.compiled_metrics.update_state(y, y_pred, sample_weight)
/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/compile_utils.py:408 update_state
metric_obj.update_state(y_t, y_p, sample_weight=mask)
/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/utils/metrics_utils.py:90 decorated
update_op = update_state_fn(*args, **kwargs)
/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/metrics.py:177 update_state_fn
return ag_update_state(*args, **kwargs)
/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/metrics.py:618 update_state **
matches = ag_fn(y_true, y_pred, **self._fn_kwargs)
TypeError: __init__() takes 2 positional arguments but 3 were given
Any idea what am I doing wrong in the custom callback which I created for roc?
Let me know what more inputs you need.

As we've mentioned in the comment, you can use built-in tf.keras.metrics.AUC while compiling the model, easy-fast-efficient. But also no problem to use either sklearn.metrics.roc_auc_score. But using it in callback may slow down your training time. Try this:
class ROAUCMetrics(tf.keras.callbacks.Callback):
def __init__(self, val_data):
super().__init__()
self.valid_x = val_data[0]
self.valid_y = val_data[1]
def on_train_begin(self, logs={}):
self.val_aucs = []
def on_epoch_end(self, epoch, logs={}):
pred = self.model.predict(self.valid_x)
val_auc = roc_auc_score(self.valid_y, pred, average='micro')
print('\nval-roc-auc: %s' % (str(round(val_auc,4))),end=100*' '+'\n')
self.val_aucs.append(val_auc)
return
# sklearn auc
roc = ROAUCMetrics(val_data=(x_val, y_val))
# tf.keras auc
model.compile(.., ..., metrics=["AUC"])
# running
model.fit(x_train, y_train, batch_size=1024,
epochs=..., callbacks=[roc],
validation_data=(x_val, y_val))
# get the values of auc, computed using sklearn auc
roc.val_aucs
However, note that both compute AUC differently, one uses Approximate AUC, another one uses Riemann sum, I've tested in one example and they pretty comparable but sometimes didn't.
Based on your 1st comment, your set up should look like this:
# (1)
# compile with no metrics - as we have custom callback metric to use
review_classifier.compile(loss='binary_crossentropy',optimizer='adam')
# (2)
# or,
# we can add another metrics e.g 'accuracy' or whatever
# here we use built-in AUC
review_classifier.compile(loss='binary_crossentropy',
optimizer='adam',metrics=['AUC'])
# sklearn auc
roc = ROAUCMetrics(val_data=(x_val, y_val))
early_stopping = EarlyStopping(patience=5)
tensorboard = TensorBoard()
reduce_lr = ReduceLROnPlateau(patience=3)
target = scoreTarget(0.90)
callbacks = [
roc,
early_stopping,
tensorboard,
reduce_lr,
target,
]
# fitting
model.fit(x_train, y_train, batch_size=1024,
epochs=..., callbacks=callbacks,
validation_data=(x_val, y_val))

Log Keras metrics for each batch as per Keras example for the loss

In the Keras doc, there is an example where a custom callback is created to log the loss for each batch. This has worked fine for me, however I also want to log metrics that I add.
For example for this code:
optimizer = Adam()
loss = losses.categorical_crossentropy
metric = ["accuracy"]
model.compile(optimizer=optimizer,
loss=loss,
metrics=metric)
class LossHistory(Callback):
def on_train_begin(self, logs={}):
self.losses = []
def on_batch_end(self, batch, logs={}):
self.losses.append(logs.get('loss'))
loss_history = LossHistory()
history = model.fit(training_data, training_labels,
batch_size=batch_size,
epochs=epochs,
verbose=2,
validation_data=(val_data, val_labels),
callbacks=[loss_history])
I can't figure out how to get access to the metrics.

The metric history is stored inside loss_history.losses:
def on_batch_end(self, batch, logs={}):
self.losses.append(logs.get('loss'))
This method will get called at the end of every batch and just appends the loss metrics into self.losses so once training has completed you can just access this list directly with loss_history.losses.
I should also add that if you wanted to include accuracy, for example, you could also do something like:
class LossHistory(Callback):
def on_train_begin(self, logs={}):
self.losses = []
self.accuracy= []
def on_batch_end(self, batch, logs={}):
self.losses.append(logs.get('loss'))
self.accuracy.append(logs.get('accuracy'))
then subsequently access it with:
loss_history.accuracy

How can I create a custom callback in Keras?

I'm interested in creating a callback while fitting my keras model. More in detail I'd like to receive a message from a bot telegram with val_acc each time an epoch is over. I know you can add a callback_list as a parameter in classifier.fit() but many callbacks are prebuilt by keras and I don't know how to add a custom one.
Thank you!

Here is an example of how I would add validation accuracy to a callback:
class AccuracyHistory(keras.callbacks.Callback):
def on_train_begin(self, logs={}):
self.acc = []
def on_epoch_end(self, batch, logs={}):
self.acc.append(logs.get('val_acc'))
history = AccuracyHistory()
model.fit(x, y,
...
callbacks=[history])

As example, I provide my custom callback with F1 metric. It calculates F1 at the end of each epoch not batch-wise but for ALL the train data passed (and optionally also validation). It can be easily customized with every other metric
class F1History(tf.keras.callbacks.Callback):
def __init__(self, train, validation=None):
super(F1History, self).__init__()
self.validation = validation
self.train = train
def on_epoch_end(self, epoch, logs={}):
logs['F1_score_train'] = float('-inf')
X_train, y_train = self.train[0], self.train[1]
y_pred = (self.model.predict(X_train).ravel()>0.5)+0
score = f1_score(y_train, y_pred)
if (self.validation):
logs['F1_score_val'] = float('-inf')
X_valid, y_valid = self.validation[0], self.validation[1]
y_val_pred = (self.model.predict(X_valid).ravel()>0.5)+0
val_score = f1_score(y_valid, y_val_pred)
logs['F1_score_train'] = np.round(score, 5)
logs['F1_score_val'] = np.round(val_score, 5)
else:
logs['F1_score_train'] = np.round(score, 5)
for fitting:
es = EarlyStopping(patience=3, verbose=1, min_delta=0.001, monitor='F1_score_val', mode='max', restore_best_weights=True)
model.fit(x_train,y_train, epochs=10,
callbacks=[F1History(train=(x_train,y_train),validation=(x_val,y_val)),es])

You can perform any action while the model is training. for this purpose keras has provided some methods as following:
on_train_begin, on_train_end, on_epoch_begin, on_epoch_end, on_test_begin,
on_test_end, on_predict_begin, on_predict_end, on_train_batch_begin, on_train_batch_end,
on_test_batch_begin, on_test_batch_end, on_predict_batch_begin,on_predict_batch_end
Example:
If you want to make predictions on the test data at the end of every epoch using the model which is being trained you can do it using the following code
class CustomCallback(keras.callbacks.Callback):
def __init__(self, model, x_test, y_test):
self.model = model
self.x_test = x_test
self.y_test = y_test
def on_epoch_end(self, epoch, logs={}):
y_pred = self.model.predict(self.x_test, self.y_test)
print('y predicted: ', y_pred)
You need mention the callback during the model.fit
model.sequence()
# your model architecture
model.fit(x_train, y_train, epochs=10,
callbacks=[CustomCallback(model, x_test, y_test)])

How to provide learning rate value to tensorboard in keras

I'm using keras and want to implement custom learning rate via keras.callbacks.LearningRateScheduler
How can I pass learning rate to be able to monitor it in tensorboard ? (keras.callbacks.TensorBoard)
Currently I have:
lrate = LearningRateScheduler(lambda epoch: initial_lr * 0.95 ** epoch)
tensorboard = TensorBoard(log_dir=LOGDIR, histogram_freq=1,
batch_size=batch_size, embeddings_freq=1,
embeddings_layer_names=embedding_layer_names )
model.fit_generator(train_generator, steps_per_epoch=n_steps,
epochs=n_epochs,
validation_data=(val_x, val_y),
callbacks=[lrate, tensorboard])

I'm not sure how to pass it to Tensorboard, but you can monitor it from python.
from keras.callbacks import Callback
class LossHistory(Callback):
def on_train_begin(self, logs={}):
self.losses = []
self.lr = []
def on_epoch_end(self, batch, logs={}):
self.losses.append(logs.get('loss'))
self.lr.append(initial_lr * 0.95 ** len(self.losses))
loss_hist = LossHistory()
Then just add loss_hist to your callbacks.
Update:
Based on this answer:
class LRTensorBoard(TensorBoard):
def __init__(self, log_dir='./logs', **kwargs):
super(LRTensorBoard, self).__init__(log_dir, **kwargs)
self.lr_log_dir = log_dir
def set_model(self, model):
self.lr_writer = tf.summary.FileWriter(self.lr_log_dir)
super(LRTensorBoard, self).set_model(model)
def on_epoch_end(self, epoch, logs=None):
lr = initial_lr * 0.95 ** epoch
summary = tf.Summary(value=[tf.Summary.Value(tag='lr',
simple_value=lr)])
self.lr_writer.add_summary(summary, epoch)
self.lr_writer.flush()
super(LRTensorBoard, self).on_epoch_end(epoch, logs)
def on_train_end(self, logs=None):
super(LRTensorBoard, self).on_train_end(logs)
self.lr_writer.close()
Just use it like the normal TensorBoard.

TensorFlow 2.x:
Creating LearningRateScheduler logs for TensorBoard can be done with the following:
from tensorflow.keras.callbacks import LearningRateScheduler, TensorBoard
# Define your scheduling function
def scheduler(epoch):
return return 0.001 * 0.95 ** epoch
# Define scheduler
lr_scheduler = LearningRateScheduler(scheduler)
# Alternatively, use an anonymous function
# lr_scheduler = LearningRateScheduler(lambda epoch: initial_lr * 0.95 ** epoch)
# Define TensorBoard callback child class
class LRTensorBoard(TensorBoard):
def __init__(self, log_dir, **kwargs):
super().__init__(log_dir, **kwargs)
self.lr_writer = tf.summary.create_file_writer(self.log_dir + '/learning')
def on_epoch_end(self, epoch, logs=None):
lr = getattr(self.model.optimizer, 'lr', None)
with self.lr_writer.as_default():
summary = tf.summary.scalar('learning_rate', lr, epoch)
super().on_epoch_end(epoch, logs)
def on_train_end(self, logs=None):
super().on_train_end(logs)
self.lr_writer.close()
# Create callback object
tensorboard_callback = LRTensorBoard(log_dir='./logs/', histogram_freq=1)
# Compile the model
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
# Train the model
r = model.fit(X_train, y_train,
validation_data=(X_val, y_val),
epochs=25, batch_size=200,
callbacks=[tensorboard_callback, lr_scheduler])
The learning rate can then be viewed in TensorBoard via
# Load the TensorBoard notebook extension
%load_ext tensorboard
#Start TensorBoard
%tensorboard --logdir ./logs