We Keep Coding

Python: tqdm progress bar stuck at 0%

I have written the following code to train a bert model on my dataset, I have used from tqdm.notebook import tqdm this import for tqdm and have used it in the loops. But when I run the program the bar stays at 0% even after the entire code has run. How to fix this?
Code
Model
TRANSFORMERS = {
"bert-multi-cased": (BertModel, BertTokenizer, "bert-base-uncased"),
}
class Transformer(nn.Module):
def __init__(self, model, num_classes=1):
"""
Constructor
Arguments:
model {string} -- Transformer to build the model on. Expects "camembert-base".
num_classes {int} -- Number of classes (default: {1})
"""
super().__init__()
self.name = model
model_class, tokenizer_class, pretrained_weights = TRANSFORMERS[model]
bert_config = BertConfig.from_json_file(MODEL_PATHS[model] + 'bert_config.json')
bert_config.output_hidden_states = True
self.transformer = BertModel(bert_config)
self.nb_features = self.transformer.pooler.dense.out_features
self.pooler = nn.Sequential(
nn.Linear(self.nb_features, self.nb_features),
nn.Tanh(),
)
self.logit = nn.Linear(self.nb_features, num_classes)
def forward(self, tokens):
"""
Usual torch forward function
Arguments:
tokens {torch tensor} -- Sentence tokens
Returns:
torch tensor -- Class logits
"""
_, _, hidden_states = self.transformer(
tokens, attention_mask=(tokens > 0).long()
)
hidden_states = hidden_states[-1][:, 0] # Use the representation of the first token of the last layer
ft = self.pooler(hidden_states)
return self.logit(ft)
Training
def fit(model, train_dataset, val_dataset, epochs=1, batch_size=8, warmup_prop=0, lr=5e-4):
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)
optimizer = AdamW(model.parameters(), lr=lr)
num_warmup_steps = int(warmup_prop * epochs * len(train_loader))
num_training_steps = epochs * len(train_loader)
scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps, num_training_steps)
loss_fct = nn.BCEWithLogitsLoss(reduction='mean').cuda()
for epoch in range(epochs):
model.train()
start_time = time.time()
optimizer.zero_grad()
avg_loss = 0
for step, (x, y_batch) in tqdm(enumerate(train_loader), total=len(train_loader)):
y_pred = model(x.to(device))
loss = loss_fct(y_pred.view(-1).float(), y_batch.float().to(device))
loss.backward()
avg_loss += loss.item() / len(train_loader)
xm.optimizer_step(optimizer, barrier=True)
#optimizer.step()
scheduler.step()
model.zero_grad()
optimizer.zero_grad()
model.eval()
preds = []
truths = []
avg_val_loss = 0.
with torch.no_grad():
for x, y_batch in tqdm(val_loader):
y_pred = model(x.to(device))
loss = loss_fct(y_pred.detach().view(-1).float(), y_batch.float().to(device))
avg_val_loss += loss.item() / len(val_loader)
probs = torch.sigmoid(y_pred).detach().cpu().numpy()
preds += list(probs.flatten())
truths += list(y_batch.numpy().flatten())
score = roc_auc_score(truths, preds)
dt = time.time() - start_time
lr = scheduler.get_last_lr()[0]
print(f'Epoch {epoch + 1}/{epochs} \t lr={lr:.1e} \t t={dt:.0f}s \t loss={avg_loss:.4f} \t val_loss={avg_val_loss:.4f} \t val_auc={score:.4f}')
model = Transformer("bert-multi-cased")
device = torch.device('cuda:2')
model = model.to(device)
epochs = 3
batch_size = 32
warmup_prop = 0.1
lr = 1e-4
train_dataset = JigsawDataset(df_train)
val_dataset = JigsawDataset(df_val)
test_dataset = JigsawDataset(df_test)
fit(model, train_dataset, val_dataset, epochs=epochs, batch_size=batch_size, warmup_prop=warmup_prop, lr=lr)
Output
0%| | 0/6986 [00:00<?, ?it/s]
How to fix this?

The import should be:
from tqdm import tqdm
The error is in the training function, correct this loop:
for x, y_batch in tqdm(val_loader, total = len(val_loader)):

Contrary to Ishan Dutta answer, tqdm.notebook.tqdm (and not tqdm.tqdm) is the correct function to use for both Jupyter Notebook and JupyterLab.
This problem can happen if you don't have ipywidgets installed or if you have installed ipywidgets before installing JupyterLab.
What fixed it for me was reinstalling ipywidgets:
pip3 uninstall ipywidgets --yes
pip3 install --upgrade ipywidgets

Keras ModelCheckpoint not saving but EarlyStopping is working fine with the same monitor argument

I've built a model and I'm using a custom function for validation. The problem is: My custom validation function is saving the validation accuracy in the logs dict, but Keras ModelCheckpoint, somehow, can't see it. EarlyStopping is working fine.
Here's the code for the validation class:
class ValidateModel(keras.callbacks.Callback):
def __init__(self, validation_data, loss_fnc):
super().__init__()
self.validation_data = validation_data
self.loss_fnc = loss_fnc
def on_epoch_end(self, epoch, logs={}):
th = 0.5
features = self.validation_data[0]
y_true = self.validation_data[1].reshape((-1,1))
y_pred = np.asarray(self.model.predict(features)).reshape((-1,1))
#Computing the validation loss.
y_true_tensor = K.constant(y_true)
y_pred_tensor = K.constant(y_pred)
val_loss = K.eval(self.loss_fnc(y_true_tensor, y_pred_tensor))
#Rounding the predicted values based on the threshold value.
#Values lesser than th are rounded to 0, while values greater than th are rounded to 1.
y_pred_rounded = y_pred / th
y_pred_rounded = np.clip(np.floor(y_pred_rounded).astype(int),0,1)
y_pred_rounded_tensor = K.constant(y_pred_rounded)
val_acc = accuracy_score(y_true, y_pred_rounded)
logs['val_loss'] = val_loss
logs['val_acc'] = val_acc
print(f'\nval_loss: {val_loss} - val_acc: {val_acc}')
And here's the function that I use to train the model:
def train_generator_model(model):
steps = int(train_df.shape[0] / TRAIN_BATCH_SIZE)
cb_validation = ValidateModel([validation_X, validation_y], iou)
cb_early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_acc',
patience=3,
mode='max',
verbose = 1)
cb_model_checkpoint = tf.keras.callbacks.ModelCheckpoint('/kaggle/working/best_generator_model.hdf5',
monitor='val_acc',
save_best_only=True,
mode='max',
verbose=1)
history = model.fit(
x = train_datagen,
epochs = 2, ##Setting to 2 to test.
callbacks = [cb_validation, cb_model_checkpoint, cb_early_stop],
verbose = 1,
steps_per_epoch = steps)
#model = tf.keras.models.load_model('/kaggle/working/best_generator_model.hdf5', custom_objects = {'iou':iou})
#model.load_weights('/kaggle/working/best_generator_model.hdf5')
return history
If I set the ModelCheckpoint parameter "save_best_model" to False, the model is saved perfectly. When the training is over and I run history.history, I can see that the val_loss is being logged, like as follows:
{'loss': [0.13096405565738678, 0.11926634609699249],
'binary_accuracy': [0.9692355990409851, 0.9716895818710327],
'val_loss': [0.23041087, 0.18325138], 'val_acc': [0.9453247578938803,
0.956172612508138]}
I'm using Tensorflow 2.3.1 and importing keras from tensorflow.
Any help is appreciated. Thank you!

I've checked the Tensorflow code and found an incompatibility between Tensorflow and Keras. Inside the tensorflow.keras.callbacks file, there's the following code:
from keras.utils import tf_utils
The problem is that there's no tf_utils in keras.utils (atleast not in Keras 2.4.3, which I was using). Strangely, no exception was thrown.
Fix #1:
Add the following code to your program:
class ModelCheckpoint_tweaked(tf.keras.callbacks.ModelCheckpoint):
def __init__(self,
filepath,
monitor='val_loss',
verbose=0,
save_best_only=False,
save_weights_only=False,
mode='auto',
save_freq='epoch',
options=None,
**kwargs):
#Change tf_utils source package.
from tensorflow.python.keras.utils import tf_utils
super(ModelCheckpoint_tweaked, self).__init__(filepath,
monitor,
verbose,
save_best_only,
save_weights_only,
mode,
save_freq,
options,
**kwargs)
And then use this new class as the ModelCheckpoint callback:
cb_model_checkpoint = ModelCheckpoint_tweaked(file_name,
monitor='val_acc',
save_best_only=True,
mode='max',
verbose=1)
Fix #2:
Update Tensorflow to version 2.4.0. If you are using a custom callback to compute the monitored parameter, add the following line to the custom callback __init__() function:
self._supports_tf_logs = True
If you don't add this line, the logs ain't gonna be persisted between the callbacks.

"Layer is not connected" issue while accessing intermediate layer from within the custom callback if model is built by sub-classing

I've a simple model and need access of intermediate layers within a custom callback to get intermediate predictions.
import tensorflow as tf
import numpy as np
X = np.ones((8,16))
y = np.sum(X, axis=1)
class CustomCallback(tf.keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs=None):
get_output = tf.keras.backend.function(
inputs = self.model.layers[0].input,
outputs = self.model.layers[1].output
)
print("\nLayer output: ", get_output(X))
If I build the model by sub-classing like below:
class Model(tf.keras.Model):
def build(self, input_shape):
self.dense1 = tf.keras.layers.Dense(units=32)
self.dense2 = tf.keras.layers.Dense(units=1)
def call(self, input_tensor):
x = self.dense1(input_tensor)
x = self.dense2(x)
return x
model = Model()
model.compile(optimizer='adam',loss='mean_squared_error', metrics='accuracy')
model.fit(X,y, epochs=2, callbacks=[CustomCallback()])
I get following error:
<ipython-input-8-bab75191182e> in on_epoch_end(self, epoch, logs)
2 def on_epoch_end(self, epoch, logs=None):
3 get_output = tf.keras.backend.function(
----> 4 inputs = self.model.layers[0].input,
5 outputs = self.model.layers[1].output
6 )
.
.
AttributeError: Layer dense is not connected, no input to return.
whereas if I build model using functional API like below, it works as expected.
initial = tf.keras.layers.Input((16,))
x = tf.keras.layers.Dense(units=32)(initial)
final = tf.keras.layers.Dense(units=1)(x)
model = tf.keras.Model(initial, final)
model.compile(optimizer='adam',loss='mean_squared_error', metrics='accuracy')
model.fit(X,y, epochs=2, callbacks=[CustomCallback()])
What's causing the error in case of model sub-classing?
TF version: 2.2.0

Log Keras metrics for each batch as per Keras example for the loss

In the Keras doc, there is an example where a custom callback is created to log the loss for each batch. This has worked fine for me, however I also want to log metrics that I add.
For example for this code:
optimizer = Adam()
loss = losses.categorical_crossentropy
metric = ["accuracy"]
model.compile(optimizer=optimizer,
loss=loss,
metrics=metric)
class LossHistory(Callback):
def on_train_begin(self, logs={}):
self.losses = []
def on_batch_end(self, batch, logs={}):
self.losses.append(logs.get('loss'))
loss_history = LossHistory()
history = model.fit(training_data, training_labels,
batch_size=batch_size,
epochs=epochs,
verbose=2,
validation_data=(val_data, val_labels),
callbacks=[loss_history])
I can't figure out how to get access to the metrics.

The metric history is stored inside loss_history.losses:
def on_batch_end(self, batch, logs={}):
self.losses.append(logs.get('loss'))
This method will get called at the end of every batch and just appends the loss metrics into self.losses so once training has completed you can just access this list directly with loss_history.losses.
I should also add that if you wanted to include accuracy, for example, you could also do something like:
class LossHistory(Callback):
def on_train_begin(self, logs={}):
self.losses = []
self.accuracy= []
def on_batch_end(self, batch, logs={}):
self.losses.append(logs.get('loss'))
self.accuracy.append(logs.get('accuracy'))
then subsequently access it with:
loss_history.accuracy

How can I create a custom callback in Keras?

I'm interested in creating a callback while fitting my keras model. More in detail I'd like to receive a message from a bot telegram with val_acc each time an epoch is over. I know you can add a callback_list as a parameter in classifier.fit() but many callbacks are prebuilt by keras and I don't know how to add a custom one.
Thank you!

Here is an example of how I would add validation accuracy to a callback:
class AccuracyHistory(keras.callbacks.Callback):
def on_train_begin(self, logs={}):
self.acc = []
def on_epoch_end(self, batch, logs={}):
self.acc.append(logs.get('val_acc'))
history = AccuracyHistory()
model.fit(x, y,
...
callbacks=[history])

As example, I provide my custom callback with F1 metric. It calculates F1 at the end of each epoch not batch-wise but for ALL the train data passed (and optionally also validation). It can be easily customized with every other metric
class F1History(tf.keras.callbacks.Callback):
def __init__(self, train, validation=None):
super(F1History, self).__init__()
self.validation = validation
self.train = train
def on_epoch_end(self, epoch, logs={}):
logs['F1_score_train'] = float('-inf')
X_train, y_train = self.train[0], self.train[1]
y_pred = (self.model.predict(X_train).ravel()>0.5)+0
score = f1_score(y_train, y_pred)
if (self.validation):
logs['F1_score_val'] = float('-inf')
X_valid, y_valid = self.validation[0], self.validation[1]
y_val_pred = (self.model.predict(X_valid).ravel()>0.5)+0
val_score = f1_score(y_valid, y_val_pred)
logs['F1_score_train'] = np.round(score, 5)
logs['F1_score_val'] = np.round(val_score, 5)
else:
logs['F1_score_train'] = np.round(score, 5)
for fitting:
es = EarlyStopping(patience=3, verbose=1, min_delta=0.001, monitor='F1_score_val', mode='max', restore_best_weights=True)
model.fit(x_train,y_train, epochs=10,
callbacks=[F1History(train=(x_train,y_train),validation=(x_val,y_val)),es])

You can perform any action while the model is training. for this purpose keras has provided some methods as following:
on_train_begin, on_train_end, on_epoch_begin, on_epoch_end, on_test_begin,
on_test_end, on_predict_begin, on_predict_end, on_train_batch_begin, on_train_batch_end,
on_test_batch_begin, on_test_batch_end, on_predict_batch_begin,on_predict_batch_end
Example:
If you want to make predictions on the test data at the end of every epoch using the model which is being trained you can do it using the following code
class CustomCallback(keras.callbacks.Callback):
def __init__(self, model, x_test, y_test):
self.model = model
self.x_test = x_test
self.y_test = y_test
def on_epoch_end(self, epoch, logs={}):
y_pred = self.model.predict(self.x_test, self.y_test)
print('y predicted: ', y_pred)
You need mention the callback during the model.fit
model.sequence()
# your model architecture
model.fit(x_train, y_train, epochs=10,
callbacks=[CustomCallback(model, x_test, y_test)])