I'm interested in creating a callback while fitting my keras model. More in detail I'd like to receive a message from a bot telegram with val_acc each time an epoch is over. I know you can add a callback_list as a parameter in classifier.fit() but many callbacks are prebuilt by keras and I don't know how to add a custom one.
Thank you!
Here is an example of how I would add validation accuracy to a callback:
class AccuracyHistory(keras.callbacks.Callback):
def on_train_begin(self, logs={}):
self.acc = []
def on_epoch_end(self, batch, logs={}):
self.acc.append(logs.get('val_acc'))
history = AccuracyHistory()
model.fit(x, y,
...
callbacks=[history])
As example, I provide my custom callback with F1 metric. It calculates F1 at the end of each epoch not batch-wise but for ALL the train data passed (and optionally also validation). It can be easily customized with every other metric
class F1History(tf.keras.callbacks.Callback):
def __init__(self, train, validation=None):
super(F1History, self).__init__()
self.validation = validation
self.train = train
def on_epoch_end(self, epoch, logs={}):
logs['F1_score_train'] = float('-inf')
X_train, y_train = self.train[0], self.train[1]
y_pred = (self.model.predict(X_train).ravel()>0.5)+0
score = f1_score(y_train, y_pred)
if (self.validation):
logs['F1_score_val'] = float('-inf')
X_valid, y_valid = self.validation[0], self.validation[1]
y_val_pred = (self.model.predict(X_valid).ravel()>0.5)+0
val_score = f1_score(y_valid, y_val_pred)
logs['F1_score_train'] = np.round(score, 5)
logs['F1_score_val'] = np.round(val_score, 5)
else:
logs['F1_score_train'] = np.round(score, 5)
for fitting:
es = EarlyStopping(patience=3, verbose=1, min_delta=0.001, monitor='F1_score_val', mode='max', restore_best_weights=True)
model.fit(x_train,y_train, epochs=10,
callbacks=[F1History(train=(x_train,y_train),validation=(x_val,y_val)),es])
You can perform any action while the model is training. for this purpose keras has provided some methods as following:
on_train_begin, on_train_end, on_epoch_begin, on_epoch_end, on_test_begin,
on_test_end, on_predict_begin, on_predict_end, on_train_batch_begin, on_train_batch_end,
on_test_batch_begin, on_test_batch_end, on_predict_batch_begin,on_predict_batch_end
Example:
If you want to make predictions on the test data at the end of every epoch using the model which is being trained you can do it using the following code
class CustomCallback(keras.callbacks.Callback):
def __init__(self, model, x_test, y_test):
self.model = model
self.x_test = x_test
self.y_test = y_test
def on_epoch_end(self, epoch, logs={}):
y_pred = self.model.predict(self.x_test, self.y_test)
print('y predicted: ', y_pred)
You need mention the callback during the model.fit
model.sequence()
# your model architecture
model.fit(x_train, y_train, epochs=10,
callbacks=[CustomCallback(model, x_test, y_test)])
Related
Currently i am working on a neural network. When i was running the script, it trained the network with no problem however, it didn't save the model to my directory, and neither the excel file with statistics.
def main(params):
ds = CIFAR10()
x_train, y_train = ds.get_train()
x_val, y_val = ds.get_val()
x_test, y_test = ds.get_test()
print(x_train.shape, np.bincount(y_train), x_val.shape, np.bincount(y_val))
model_holder = WideResidualNetwork(preprocess_mode='FS')
model = model_holder.build_model(ds.get_input_shape(), ds.get_nb_classes())
loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
#metrics = ['sparse_categorical_accuracy', tf.keras.metrics.categorical_accuracy]
metrics = ['sparse_categorical_accuracy']
#model.compile(tf.keras.optimizers.SGD(learning_rate=0.1, momentum=0.9, nesterov=True, name='SGD'), loss_fn, metrics)
model.compile(tfa.optimizers.SGDW(learning_rate=0.1, momentum=0.9, weight_decay=0.0005, nesterov=True, name='SGD'), loss_fn, metrics)
if params.aug:
params.save_dir = os.path.join(params.save_dir, 'aug')
m_path = os.path.join(params.save_dir, model_holder.get_name())
util.mk_parent_dir(m_path)
callbacks = [tf.keras.callbacks.ModelCheckpoint(m_path + '_{epoch:03d}-{val_loss:.2f}-{val_accuracy:.2f}.h5'),
tf.keras.callbacks.CSVLogger(os.path.join(params.save_dir, 'metrics.csv'))]
train_dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))
train_dataset = train_dataset.shuffle(buffer_size=x_train.shape[0], reshuffle_each_iteration=True)
if params.aug:
AUTOTUNE = tf.data.experimental.AUTOTUNE
flipping = True
img_size = x_train.shape[1:]
aug_fn = lambda x, y: augment(x, y, image_size=img_size, flipping=flipping)
train_dataset = train_dataset.map(aug_fn, num_parallel_calls=AUTOTUNE)
train_dataset = train_dataset.batch(params.batch_size).prefetch(10)
lr_steps = [60,120,160,200]
decay = 0.2
With this part of the script, i had no problem, however in the script below i needed to lower the learning rate in certain epoch's.
The only problem i have is with the "cb.on_epoch_end(i, stats)" function. It does not save the model, neither the stats. When i pass a dictionary it says 'NoneType' object has no attribute 'distribute_strategy' even while i have no key like that, how can it be NoneType.
for i in range(params.epoch):
for cb in callbacks:
cb.on_epoch_begin(i)
hist_i = []
if lr_steps is not None and np.isin(i+1, lr_steps):
K.set_value(model.optimizer.lr, K.get_value(model.optimizer.lr) * decay)
K.set_value(model.optimizer.weight_decay, K.get_value(model.optimizer.weight_decay) * decay)
for step, (x_batch_train, y_batch_train) in enumerate(train_dataset):
batch_eval = model.train_on_batch(x_batch_train,y_batch_train)
hist_i.append(batch_eval)
vala_eval = model.evaluate(x_val, y_val, verbose=0)
train_eval = np.mean(hist_i, axis=0)
stats = {'loss': train_eval[0], 'accuracy': train_eval[1], 'val_loss': vala_eval[0], 'val_accuracy': vala_eval[1]}
for cb in callbacks:
cb.on_epoch_end(i, stats)
if i == (params.epoch - 1) or model.stop_training:
cb.on_train_end()
print(model.evaluate(x_val,y_val))
The only goal is to save the model, with the statistics in an excel file. All help is appreciated!
I've built a model and I'm using a custom function for validation. The problem is: My custom validation function is saving the validation accuracy in the logs dict, but Keras ModelCheckpoint, somehow, can't see it. EarlyStopping is working fine.
Here's the code for the validation class:
class ValidateModel(keras.callbacks.Callback):
def __init__(self, validation_data, loss_fnc):
super().__init__()
self.validation_data = validation_data
self.loss_fnc = loss_fnc
def on_epoch_end(self, epoch, logs={}):
th = 0.5
features = self.validation_data[0]
y_true = self.validation_data[1].reshape((-1,1))
y_pred = np.asarray(self.model.predict(features)).reshape((-1,1))
#Computing the validation loss.
y_true_tensor = K.constant(y_true)
y_pred_tensor = K.constant(y_pred)
val_loss = K.eval(self.loss_fnc(y_true_tensor, y_pred_tensor))
#Rounding the predicted values based on the threshold value.
#Values lesser than th are rounded to 0, while values greater than th are rounded to 1.
y_pred_rounded = y_pred / th
y_pred_rounded = np.clip(np.floor(y_pred_rounded).astype(int),0,1)
y_pred_rounded_tensor = K.constant(y_pred_rounded)
val_acc = accuracy_score(y_true, y_pred_rounded)
logs['val_loss'] = val_loss
logs['val_acc'] = val_acc
print(f'\nval_loss: {val_loss} - val_acc: {val_acc}')
And here's the function that I use to train the model:
def train_generator_model(model):
steps = int(train_df.shape[0] / TRAIN_BATCH_SIZE)
cb_validation = ValidateModel([validation_X, validation_y], iou)
cb_early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_acc',
patience=3,
mode='max',
verbose = 1)
cb_model_checkpoint = tf.keras.callbacks.ModelCheckpoint('/kaggle/working/best_generator_model.hdf5',
monitor='val_acc',
save_best_only=True,
mode='max',
verbose=1)
history = model.fit(
x = train_datagen,
epochs = 2, ##Setting to 2 to test.
callbacks = [cb_validation, cb_model_checkpoint, cb_early_stop],
verbose = 1,
steps_per_epoch = steps)
#model = tf.keras.models.load_model('/kaggle/working/best_generator_model.hdf5', custom_objects = {'iou':iou})
#model.load_weights('/kaggle/working/best_generator_model.hdf5')
return history
If I set the ModelCheckpoint parameter "save_best_model" to False, the model is saved perfectly. When the training is over and I run history.history, I can see that the val_loss is being logged, like as follows:
{'loss': [0.13096405565738678, 0.11926634609699249],
'binary_accuracy': [0.9692355990409851, 0.9716895818710327],
'val_loss': [0.23041087, 0.18325138], 'val_acc': [0.9453247578938803,
0.956172612508138]}
I'm using Tensorflow 2.3.1 and importing keras from tensorflow.
Any help is appreciated. Thank you!
I've checked the Tensorflow code and found an incompatibility between Tensorflow and Keras. Inside the tensorflow.keras.callbacks file, there's the following code:
from keras.utils import tf_utils
The problem is that there's no tf_utils in keras.utils (atleast not in Keras 2.4.3, which I was using). Strangely, no exception was thrown.
Fix #1:
Add the following code to your program:
class ModelCheckpoint_tweaked(tf.keras.callbacks.ModelCheckpoint):
def __init__(self,
filepath,
monitor='val_loss',
verbose=0,
save_best_only=False,
save_weights_only=False,
mode='auto',
save_freq='epoch',
options=None,
**kwargs):
#Change tf_utils source package.
from tensorflow.python.keras.utils import tf_utils
super(ModelCheckpoint_tweaked, self).__init__(filepath,
monitor,
verbose,
save_best_only,
save_weights_only,
mode,
save_freq,
options,
**kwargs)
And then use this new class as the ModelCheckpoint callback:
cb_model_checkpoint = ModelCheckpoint_tweaked(file_name,
monitor='val_acc',
save_best_only=True,
mode='max',
verbose=1)
Fix #2:
Update Tensorflow to version 2.4.0. If you are using a custom callback to compute the monitored parameter, add the following line to the custom callback __init__() function:
self._supports_tf_logs = True
If you don't add this line, the logs ain't gonna be persisted between the callbacks.
In the Keras doc, there is an example where a custom callback is created to log the loss for each batch. This has worked fine for me, however I also want to log metrics that I add.
For example for this code:
optimizer = Adam()
loss = losses.categorical_crossentropy
metric = ["accuracy"]
model.compile(optimizer=optimizer,
loss=loss,
metrics=metric)
class LossHistory(Callback):
def on_train_begin(self, logs={}):
self.losses = []
def on_batch_end(self, batch, logs={}):
self.losses.append(logs.get('loss'))
loss_history = LossHistory()
history = model.fit(training_data, training_labels,
batch_size=batch_size,
epochs=epochs,
verbose=2,
validation_data=(val_data, val_labels),
callbacks=[loss_history])
I can't figure out how to get access to the metrics.
The metric history is stored inside loss_history.losses:
def on_batch_end(self, batch, logs={}):
self.losses.append(logs.get('loss'))
This method will get called at the end of every batch and just appends the loss metrics into self.losses so once training has completed you can just access this list directly with loss_history.losses.
I should also add that if you wanted to include accuracy, for example, you could also do something like:
class LossHistory(Callback):
def on_train_begin(self, logs={}):
self.losses = []
self.accuracy= []
def on_batch_end(self, batch, logs={}):
self.losses.append(logs.get('loss'))
self.accuracy.append(logs.get('accuracy'))
then subsequently access it with:
loss_history.accuracy
So I'm having an issue and I'm stuck on it for a while now
I'm trying to understand and repeat a jupyter tutorial about adversarial example.
However, when I'm loading a model, depending on how is it done I have big differences between the two models accuracy:
- when it's loading inside a class, there is no issues
- when it's loading on jupyter, the accuracy decrease
The class use the following lines:
from keras.models import Sequential, load_model
class Lenet:
def __init__(self, epochs=200, batch_size=128, load_weights=True):
self.name = 'lenet'
self.model_filename = 'networks/models/lenet.h5'
self.num_classes = 10
self.input_shape = 32, 32, 3
self.batch_size = batch_size
self.epochs = epochs
self.iterations = 391
self.weight_decay = 0.0001
self.log_filepath = r'networks/models/lenet/'
if load_weights:
try:
self._model = load_model(self.model_filename)
print('Successfully loaded', self.name)
except (ImportError, ValueError, OSError):
print('Failed to load', self.name)
what I am doing is:
modelPath = "networks/models/lenet.h5"
lenet2 = keras.models.load_model(modelPath)
modelPath2 = "networks/models/resnet.h5"
resnet2 = keras.models.load_model(modelPath2)
However, when I'm testing the accuracy of the two models:
lenet = LeNet()
resnet = ResNet() #the class is nearly the same as LeNet
models = [lenet, lenet2, resnet, resnet2]
network_stats, correct_imgs = helper.evaluate_models(models, x_test, y_test)
network_stats = pd.DataFrame(network_stats, columns=['name', 'accuracy', 'param_count'])
I'm getting this following result: (network_stats)
name accuracy param_count
0 lenet 0.7488 62006
1 sequential_1 0.4800 62006
2 resnet 0.9231 470218
3 model_1 0.1092 470218
link to the picture
to explain the picture: on the left, this is the class lenet, that give good results. On the right, my failed try to load the same model
Do you have any idea about why is that happening?
--edit--
can't post images yet on stackoverflow, so I explained a little bit more the issue
--edit2--
can reproduce with another network (resnet)
Can you provide the link to the tutorial you are following?
Are you training the model or just testing the accuracy?
My guess... the class has a color preprocessing function that is transforming the input in the "accuracy" method. If the non-class model is not using that same function then that's about the outcome discrepancy I would expect to see.
def accuracy(self):
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
y_train = keras.utils.to_categorical(y_train, self.num_classes)
y_test = keras.utils.to_categorical(y_test, self.num_classes)
# color preprocessing
x_train, x_test = self.color_preprocessing(x_train, x_test) # <--- here
return self._model.evaluate(x_test, y_test, verbose=0)[1]
I'm using keras and want to implement custom learning rate via keras.callbacks.LearningRateScheduler
How can I pass learning rate to be able to monitor it in tensorboard ? (keras.callbacks.TensorBoard)
Currently I have:
lrate = LearningRateScheduler(lambda epoch: initial_lr * 0.95 ** epoch)
tensorboard = TensorBoard(log_dir=LOGDIR, histogram_freq=1,
batch_size=batch_size, embeddings_freq=1,
embeddings_layer_names=embedding_layer_names )
model.fit_generator(train_generator, steps_per_epoch=n_steps,
epochs=n_epochs,
validation_data=(val_x, val_y),
callbacks=[lrate, tensorboard])
I'm not sure how to pass it to Tensorboard, but you can monitor it from python.
from keras.callbacks import Callback
class LossHistory(Callback):
def on_train_begin(self, logs={}):
self.losses = []
self.lr = []
def on_epoch_end(self, batch, logs={}):
self.losses.append(logs.get('loss'))
self.lr.append(initial_lr * 0.95 ** len(self.losses))
loss_hist = LossHistory()
Then just add loss_hist to your callbacks.
Update:
Based on this answer:
class LRTensorBoard(TensorBoard):
def __init__(self, log_dir='./logs', **kwargs):
super(LRTensorBoard, self).__init__(log_dir, **kwargs)
self.lr_log_dir = log_dir
def set_model(self, model):
self.lr_writer = tf.summary.FileWriter(self.lr_log_dir)
super(LRTensorBoard, self).set_model(model)
def on_epoch_end(self, epoch, logs=None):
lr = initial_lr * 0.95 ** epoch
summary = tf.Summary(value=[tf.Summary.Value(tag='lr',
simple_value=lr)])
self.lr_writer.add_summary(summary, epoch)
self.lr_writer.flush()
super(LRTensorBoard, self).on_epoch_end(epoch, logs)
def on_train_end(self, logs=None):
super(LRTensorBoard, self).on_train_end(logs)
self.lr_writer.close()
Just use it like the normal TensorBoard.
TensorFlow 2.x:
Creating LearningRateScheduler logs for TensorBoard can be done with the following:
from tensorflow.keras.callbacks import LearningRateScheduler, TensorBoard
# Define your scheduling function
def scheduler(epoch):
return return 0.001 * 0.95 ** epoch
# Define scheduler
lr_scheduler = LearningRateScheduler(scheduler)
# Alternatively, use an anonymous function
# lr_scheduler = LearningRateScheduler(lambda epoch: initial_lr * 0.95 ** epoch)
# Define TensorBoard callback child class
class LRTensorBoard(TensorBoard):
def __init__(self, log_dir, **kwargs):
super().__init__(log_dir, **kwargs)
self.lr_writer = tf.summary.create_file_writer(self.log_dir + '/learning')
def on_epoch_end(self, epoch, logs=None):
lr = getattr(self.model.optimizer, 'lr', None)
with self.lr_writer.as_default():
summary = tf.summary.scalar('learning_rate', lr, epoch)
super().on_epoch_end(epoch, logs)
def on_train_end(self, logs=None):
super().on_train_end(logs)
self.lr_writer.close()
# Create callback object
tensorboard_callback = LRTensorBoard(log_dir='./logs/', histogram_freq=1)
# Compile the model
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
# Train the model
r = model.fit(X_train, y_train,
validation_data=(X_val, y_val),
epochs=25, batch_size=200,
callbacks=[tensorboard_callback, lr_scheduler])
The learning rate can then be viewed in TensorBoard via
# Load the TensorBoard notebook extension
%load_ext tensorboard
#Start TensorBoard
%tensorboard --logdir ./logs