We Keep Coding

TypeError: forward() got an unexpected keyword argument 'baseline value'. How do I correctly load a saved model in Skorch?

I saved my Skorch neural net model using the below code:
net_b = NeuralNetClassifier(
Classifier_b,
max_epochs=50,
optimizer__momentum= 0.9,
lr=0.1,
device=device,
)
#Fit the model on the full data
net_b.fit(merged_X_train, merged_Y_train);
#Test saving
import pickle
with open('MLP.pkl', 'wb') as f:
pickle.dump(net_b, f)
When I try to load this model again and run it against test data, I receive the following error:
TypeError: forward() got an unexpected keyword argument 'baseline value'
This is my code:
#Split the data
X_train, y_train, X_valid, y_valid,X_test, y_test = train_valid_test_split(rescaled_data, target = 'fetal_health',
train_size=0.8, valid_size=0.1, test_size=0.1)
input_dim = f_df_toscale.shape[1]
output_dim = len(np.unique(f_target))
hidden_dim_a = 20
hidden_dim_b = 12
device = 'cpu'
class Classifier_b(nn.Module):
def __init__(self,
input_dim = input_dim,
hidden_dim_a = hidden_dim_b,
output_dim = output_dim):
super(Classifier_b, self).__init__()
#Take the inputs and pass these to a hidden layer
self.hidden = nn.Linear(input_dim,hidden_dim_b)
#Take the hidden layer and pass it through an additional hidden layer
self.hidden_b = nn.Linear(hidden_dim_a,hidden_dim_b)
#Take the hidden layer and pass to a multi nerouon output
self.output = nn.Linear(hidden_dim_b,output_dim)
def forward(self, x):
hidden = F.relu(self.hidden(x))
hidden = F.relu(self.hidden_b(hidden))
output = F.softmax(self.output(hidden))
return output
#load the model
with open('MLP.pkl', 'rb') as f:
model_MLP = pickle.load(f)
#Test the model
y_pred = model_MLP.predict(X_test)
ML = accuracy_score(y_test, y_pred)
print('The accuracy score for the MLP is ', ML)
When I run this model normally in the original notebook everything run fines. But when I try to load my model from a saved state I get the error. Any idea why? I have nothing called 'baseline value'.
Thanks

The save and load model can be problematic if the code changes. So it is better to use
save_params() and load_params()
In your case
net_b.save_params(f_params='some-file.pkl')
To load the model first initialize (initializing is very important) and then load parameters
new_net.initialize()
new_net.load_params(f_params='some-file.pkl')

Train model after load in MLFlow

My goal is to store an empty model into MLFlow Registry and then load it for training.
I have a register_model.py which looks like this:
if __name__ == "__main__":
remote_server_uri = "http://127.0.0.1:5000"
mlflow.set_tracking_uri(remote_server_uri)
# Load and compile Keras model
model = tf.keras.applications.MobileNetV2((32, 32, 3), classes=10, weights=None)
model.compile("adam", "sparse_categorical_crossentropy", metrics=["accuracy"])
# Load CIFAR-10 dataset
(x_train, y_train), (x_test, y_test) = tf.keras.datasets.cifar10.load_data()
epochs = 1
batch_size = 32
mlflow.tensorflow.autolog()
model.fit(x_train, y_train, epochs=epochs, batch_size=batch_size)
tf.keras.models.save_model(model, "models/tensorflow")
mlflow.tensorflow.log_model(
tf_saved_model_dir='models/tensorflow',
tf_meta_graph_tags=None,
tf_signature_def_key='serving_default',
artifact_path="saved/models/tensorflow",
registered_model_name="tensorflow-MobileNetV2-32inputs"
)
Then I'm trying to load it using:
if __name__ == "__main__":
remote_server_uri = "http://127.0.0.1:5000"
mlflow.set_tracking_uri(remote_server_uri)
model_name = "tensorflow-MobileNetV2-32inputs"
model_version = 1
model = mlflow.tensorflow.load_model(
model_uri=f"models:/{model_name}/{model_version}"
)
I would expect my model to be a able to do things like 'model.fit()' and 'model.predict()' but Im always getting:
AttributeError: '_WrapperFunction' object has no attribute 'fit'
So my question is: it is possible to save a tensorflow/keras model and to load the architecture to be trained/retrained and even modified through mlflow? Load the model, add a new layer, store the model either as a new version or as a new model itself, for example.
Thanks in advance!

Tensorflow ValueError: Unexpected result of `predict_function` (Empty batch_outputs). Please use `Model.compile(..., run_eagerly=True)`

I'm trying to train U-NET using Brats dataset on colab, but it showing the following error :
ValueError: Unexpected result of train_function (Empty logs). Please use Model.compile(..., run_eagerly=True), or tf.config.run_functions_eagerly(True) for more information of where went wrong, or file a issue/bug to tf.keras
the code for training:
import os
import tables
import numpy as np
from config import cfg
from model import unet_model
from data_generator import CustomDataGenerator
from tensorflow.keras.callbacks import CSVLogger, ModelCheckpoint, TensorBoard
def train_model(hdf5_dir, brains_idx_dir, view, modified_unet=True, batch_size=16, val_batch_size=32,
lr=0.01, epochs=100, hor_flip=False, ver_flip=False, zoom_range=0.0, save_dir='./save/',
start_chs=64, levels=3, multiprocessing=False, load_model_dir=None):
"""
The function that builds/loads UNet model, initializes the data generators for training and validation, and finally
trains the model.
"""
# preparing generators
hdf5_file = tables.open_file(hdf5_dir, mode='r+')
brain_idx = np.load(brains_idx_dir)
datagen_train = CustomDataGenerator(hdf5_file, brain_idx, batch_size, view, 'train',
hor_flip, ver_flip, zoom_range, shuffle=True)
datagen_val = CustomDataGenerator(hdf5_file, brain_idx, val_batch_size, view, 'validation', shuffle=False)
# add callbacks
save_dir = os.path.join(save_dir, '{}_{}'.format(view, os.path.basename(brains_idx_dir)[:5]))
if not os.path.isdir(save_dir):
os.mkdir(save_dir)
logger = CSVLogger(os.path.join(save_dir, 'log.txt'))
checkpointer = ModelCheckpoint(filepath = os.path.join(save_dir, 'model.hdf5'), verbose=1, save_best_only=True)
tensorboard = TensorBoard(os.path.join(save_dir, 'tensorboard'))
callbacks = [logger, checkpointer, tensorboard]
# building the model
model_input_shape = datagen_train.data_shape[1:]
model = unet_model(model_input_shape, modified_unet, lr, start_chs, levels)
# training the model
model.fit_generator(datagen_train, epochs=epochs, use_multiprocessing=multiprocessing,
callbacks=callbacks, validation_data = datagen_val)
if __name__ == '__main__':
train_model(cfg['hdf5_dir'], cfg['brains_idx_dir'], cfg['view'], cfg['modified_unet'], cfg['batch_size'],
cfg['val_batch_size'], cfg['lr'], cfg['epochs'], cfg['hor_flip'], cfg['ver_flip'], cfg['zoom_range'],
cfg['save_dir'], cfg['start_chs'], cfg['levels'], cfg['multiprocessing'],
cfg['load_model_dir'])
and the error msg is :
Why am I getting the above error and how can I fix it?

Keras tuner and TPU in Google Colab

I have some problems with keras tuner and tpu. When I run the code below, everything works well and network training is fast.
vocab_size = 5000
embedding_dim = 64
max_length = 2000
def create_model():
model = tf.keras.Sequential([
tf.keras.layers.Embedding(vocab_size, embedding_dim),
tf.keras.layers.LSTM(100, dropout=0.5, recurrent_dropout=0.5),
tf.keras.layers.Dense(embedding_dim, activation='relu'),
tf.keras.layers.Dense(4, activation='softmax')
])
return model
resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='grpc://' + os.environ['COLAB_TPU_ADDR'])
tf.config.experimental_connect_to_cluster(resolver)
tf.tpu.experimental.initialize_tpu_system(resolver)
strategy = tf.distribute.experimental.TPUStrategy(resolver)
with strategy.scope():
model = create_model()
model.compile(optimizer='adam',
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=['sparse_categorical_accuracy'])
model.fit(train_padded, y_train,
epochs=10,
validation_split=0.15,
verbose=1, batch_size=128)
When I use a keras tuner, the neural network learns slowly. I believe that TPU is not used.
vocab_size = 5000
max_length = 2000
resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='grpc://' + os.environ['COLAB_TPU_ADDR'])
tf.config.experimental_connect_to_cluster(resolver)
tf.tpu.experimental.initialize_tpu_system(resolver)
strategy = tf.distribute.experimental.TPUStrategy(resolver)
def build_model(hp):
model = tf.keras.Sequential()
activation_choice = hp.Choice('activation', values=['relu', 'sigmoid', 'tanh', 'elu', 'selu'])
embedding_dim = hp.Int('units_hidden', min_value=128, max_value=24, step=8)
model.add(tf.keras.layers.Embedding(vocab_size, embedding_dim))
model.add(tf.keras.layers.LSTM(hp.Int('LSTM_Units', min_value=50, max_value=500, step=10),
dropout=hp.Float('dropout', 0, 0.5, step=0.1, default=0),
recurrent_dropout=hp.Float('recurrent_dropout', 0, 0.5, step=0.1, default=0)))
model.add(tf.keras.layers.Dense(embedding_dim, activation=activation_choice))
model.add(tf.keras.layers.Dense(4, activation='softmax'))
model.compile(
optimizer=hp.Choice('optimizer', values=['adam', 'rmsprop', 'SGD']),
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=['sparse_categorical_accuracy'])
return model
with strategy.scope():
tuner = Hyperband(
build_model,
objective='val_accuracy',
max_epochs=10,
hyperband_iterations=2)
tuner.search(train_padded, y_train,
batch_size=128,
epochs=10,
callbacks=[EarlyStopping(patience=1)],
validation_split=0.15,
verbose=1)
best_models = tuner.get_best_models(1)
best_model.save('/content/drive/My Drive/best_model.h5')
Notebook link
How to make a keras tuner work with TPU?

You need to pass it to the tuner:
tuner = Hyperband(
build_model,
objective='val_accuracy',
max_epochs=10,
hyperband_iterations=2,
distribution_strategy=strategy,)
(and remove the strategy.scope() part)

To add ...
I don't use Google Colab, but Kaggle. Using TPU, I get that same error "File system scheme '[local]' not implemented", when the tuner tries to write the checkpoints on Kaggle's working directory.
Since I don't have a gs://location, I just "modified" the function called by Keras Tuner to save checkpoints, to allow writing to local dir, which is the Kaggle working directory. I used patch() to mock the function.
First important thing is that Keras Tuner must be version 1.1.2 and above.
Example:
from mock import patch
<your code>
# now the new function to "replace" the existing one (keras_tuner.engine.tuner_utils.SaveBestEpoch.on_epoch_end)
def new_on_epoch_end(self, epoch, logs=None):
if not self.objective.has_value(logs):
# Save on every epoch if metric value is not in the logs. Either no
# objective is specified, or objective is computed and returned
# after `fit()`.
#***** the following are the lines I added ******************************************
# Save model in Tensorflow's "SavedModel" format
save_locally = tf.saved_model.SaveOptions(experimental_io_device = '/job:localhost')
# I then added ', options = save_locally' to the line below.
#************************************************************************************
self.model.save_weights(self.filepath, options = save_locally)
return
current_value = self.objective.get_value(logs)
if self.objective.better_than(current_value, self.best_value):
self.best_value = current_value
#***** the following are the lines I added ******************************************
# Save model in Tensorflow's "SavedModel" format
save_locally = tf.saved_model.SaveOptions(experimental_io_device = '/job:localhost')
# I then added ', options = save_locally' to the line below.
#************************************************************************************
self.model.save_weights(self.filepath, options = save_locally)
with patch('keras_tuner.engine.tuner_utils.SaveBestEpoch.on_epoch_end', new_on_epoch_end):
# Perform hypertuning. The parameters are exactly like those in the fit() method.
tuner.search(
X_train,
y_train,
epochs=num_of_epochs,
validation_data = (X_valid, y_valid),
callbacks=[early_stopping]
)
<more of your code>
Since I used 'with patch', after all is done, it reverts back to the original code automatically.
I hope this will be useful for those using Kaggle, or those who want to write to a local dir.

How can I create a custom callback in Keras?

I'm interested in creating a callback while fitting my keras model. More in detail I'd like to receive a message from a bot telegram with val_acc each time an epoch is over. I know you can add a callback_list as a parameter in classifier.fit() but many callbacks are prebuilt by keras and I don't know how to add a custom one.
Thank you!

Here is an example of how I would add validation accuracy to a callback:
class AccuracyHistory(keras.callbacks.Callback):
def on_train_begin(self, logs={}):
self.acc = []
def on_epoch_end(self, batch, logs={}):
self.acc.append(logs.get('val_acc'))
history = AccuracyHistory()
model.fit(x, y,
...
callbacks=[history])

As example, I provide my custom callback with F1 metric. It calculates F1 at the end of each epoch not batch-wise but for ALL the train data passed (and optionally also validation). It can be easily customized with every other metric
class F1History(tf.keras.callbacks.Callback):
def __init__(self, train, validation=None):
super(F1History, self).__init__()
self.validation = validation
self.train = train
def on_epoch_end(self, epoch, logs={}):
logs['F1_score_train'] = float('-inf')
X_train, y_train = self.train[0], self.train[1]
y_pred = (self.model.predict(X_train).ravel()>0.5)+0
score = f1_score(y_train, y_pred)
if (self.validation):
logs['F1_score_val'] = float('-inf')
X_valid, y_valid = self.validation[0], self.validation[1]
y_val_pred = (self.model.predict(X_valid).ravel()>0.5)+0
val_score = f1_score(y_valid, y_val_pred)
logs['F1_score_train'] = np.round(score, 5)
logs['F1_score_val'] = np.round(val_score, 5)
else:
logs['F1_score_train'] = np.round(score, 5)
for fitting:
es = EarlyStopping(patience=3, verbose=1, min_delta=0.001, monitor='F1_score_val', mode='max', restore_best_weights=True)
model.fit(x_train,y_train, epochs=10,
callbacks=[F1History(train=(x_train,y_train),validation=(x_val,y_val)),es])

You can perform any action while the model is training. for this purpose keras has provided some methods as following:
on_train_begin, on_train_end, on_epoch_begin, on_epoch_end, on_test_begin,
on_test_end, on_predict_begin, on_predict_end, on_train_batch_begin, on_train_batch_end,
on_test_batch_begin, on_test_batch_end, on_predict_batch_begin,on_predict_batch_end
Example:
If you want to make predictions on the test data at the end of every epoch using the model which is being trained you can do it using the following code
class CustomCallback(keras.callbacks.Callback):
def __init__(self, model, x_test, y_test):
self.model = model
self.x_test = x_test
self.y_test = y_test
def on_epoch_end(self, epoch, logs={}):
y_pred = self.model.predict(self.x_test, self.y_test)
print('y predicted: ', y_pred)
You need mention the callback during the model.fit
model.sequence()
# your model architecture
model.fit(x_train, y_train, epochs=10,
callbacks=[CustomCallback(model, x_test, y_test)])