I want to change the get_gradients method of Adam to execute gradient Centralization, the following code is from https://keras.io/examples/vision/gradient_centralization/
My question is why the code print('ttt') in method get_gradients prints nothing when fit function is executing?
print('dddd') works and print dddd in screen.
from tensorflow.keras.optimizers import Adam
import tensorflow as tf
class GCAdam(Adam):
def __init__(self, learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-7, amsgrad=False, name='Adam', **kwargs):
super().__init__(learning_rate=learning_rate, beta_1=beta_1, beta_2=beta_2, epsilon=epsilon, amsgrad=amsgrad, name=name, **kwargs)
print('dddd')
def get_gradients(self, loss, params):
# We here just provide a modified get_gradients() function since we are
# trying to just compute the centralized gradients.
grads = []
gradients = super().get_gradients()
print('ttt')
for grad in gradients:
grad_len = len(grad.shape)
if grad_len > 1:
axis = list(range(grad_len - 1))
grad -= tf.reduce_mean(grad, axis=axis, keep_dims=True)
grads.append(grad)
return grads
optimizergc = GCAdam(learning_rate=1e-4)
model.compile(loss="categorical_crossentropy", optimizer=optimizergc, metrics=["accuracy"])
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_split=0.1,verbose=1)
Related
I am trying to implement a Multivariate Time-series using pytorch. Here i am giving only that part of the code where i am getting the error, i have included all the mentioned classes in my full code though. The code has been referred from https://towardsdatascience.com/building-rnn-lstm-and-gru-for-time-series-using-pytorch-a46e5b094e7b
class Optimization:
def __init__(self, model, loss_fn, optimizer):
self.model = model
self.loss_fn = loss_fn
self.optimizer = optimizer
self.train_losses = []
self.val_losses = []
def train_step(self, x, y):
# Sets model to train mode
self.model.train()
# Makes predictions
yhat = self.model(x)
# Computes loss
loss = self.loss_fn(y, yhat)
# Computes gradients
loss.backward()
# Updates parameters and zeroes gradients
self.optimizer.step()
self.optimizer.zero_grad()
# Returns the loss
return loss.item()
import torch.optim as optim
def get_model(model, model_params):
models = {
"rnn": RNNModel,
"lstm": LSTMModel,
"gru": GRUModel,
}
return models.get(model.lower())(**model_params)
input_dim = len(X_train.columns)
output_dim = 1
hidden_dim = 64
layer_dim = 3
batch_size = 64
dropout = 0.2
n_epochs = 100
learning_rate = 1e-3
weight_decay = 1e-6
model_params = {'input_dim': input_dim,
'hidden_dim' : hidden_dim,
'layer_dim' : layer_dim,
'output_dim' : output_dim,
'dropout_prob' : dropout}
model = get_model('lstm', model_params)
loss_fn = nn.MSELoss(reduction="mean")
optimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)
opt = Optimization(model=model, loss_fn=loss_fn, optimizer=optimizer)
opt.train(train_loader, val_loader, batch_size=batch_size,
n_epochs=n_epochs, n_features=input_dim)
opt.plot_losses()
predictions, values = opt.evaluate(test_loader_one, batch_size=1, n_features=input_dim)
Is giving me this error.
AttributeError Traceback (most recent call last)
e:\codefolder\multivar.ipynb Cell 21' in <cell line: 25>()
22 optimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)
24 opt = Optimization(model=model, loss_fn=loss_fn, optimizer=optimizer)
---> 25 opt.train(train_loader, val_loader, batch_size=batch_size, n_epochs=n_epochs, n_features=input_dim)
26 opt.plot_losses()
28 predictions, values = opt.evaluate(test_loader_one, batch_size=1, n_features=input_dim)
AttributeError: 'Optimization' object has no attribute 'train'
P.S.- I have included all the classes and codes into my code as given in the article to which i referred to. Because of space and length of the codes i have included only the main code where i am getting the error.
You've written
opt = Optimization(...)
Without either importing or defining that class. Python is telling you it doesn't recognize that object or its methods
I found torch.optim.Optimizer here, is it possible that's what you're looking for?
I'm trying to implement a bespoke loss calculation model but keep getting hit with recognition failures. eg: "ValueError: Unknown loss function: root_mean_squared_error_fraction. Please ensure this object is passed to the custom_objects argument."
The function "root_mean_squared_fraction" exists and is functioning, I know this because I call it elsewhere outside of the Keras model and it functions as expected. I'm clearly not understanding something about injecting this into the model definition and would appreciate any advice? Thanks.
from keras.models import load_model
from keras import backend as K
def root_mean_squared_error_fraction(y_true, y_pred):
return K.sqrt(K.mean(K.square((y_pred - y_true)/y_true)))
This is my model routine which does work when the =rmsef in the model.compile is replaced with ='mse':
def ResModel(Svect, Xvect, Yvect, dtrain):
model = Sequential()
model.add(LSTM(64, activation='relu',\
input_shape=(Xvect.shape[1], Xvect.shape[2]),\
return_sequences=True))
model.add(LSTM(32, activation='relu', return_sequences=False))
model.add(Dropout(0.2))
model.add(Dense(Yvect.shape[1]))
rmsef = root_mean_squared_error_fraction
model.compile(optimizer = "adam", loss = rmsef )
bmfile = 'bestmodel.h5'
earlystop = EarlyStopping(monitor='val_loss', mode='auto',\
verbose=0, patience=mpat, min_delta=0.005)
chkpoint = ModelCheckpoint(bmfile, monitor = 'val_loss', mode = 'auto',\
save_best_only = True )
history = model.fit(Xvect, Yvect, epochs=mcycl, batch_size=32,\
validation_split=dsplit, verbose=0,\
callbacks = [earlystop, chkpoint] )
saved = load_model('bestmodel.h5')
score = saved.evaluate(Xvect, Yvect, verbose=0)
print("Epoch: %04d yeilded best fit with overall loss of: %0.4f "\
% ((earlystop.stopped_epoch + 1), score ) )
Yvect = descalevector(Svect, saved.predict(Xvect),dtrain )
return Yvect, score
I am trying to implement custom roccallback for keras. The call back function I wrote is below. I have to achieve a target of 0.90.
AUROC callback:
class rocback(Callback):
def __init__(self, validation_data):
super(rocback, self).__init__()
# self.training_data = training_data
self.validation_data = validation_data
def on_train_begin(self, logs={}) :
return
def on_epoch_end(self, epoch, logs={}):
probs = self.model.predict(self.validation_data[0])
probs = np.round(probs)
y_true = self.validation_data[1]
y_true = np.round(y_true)
score = roc_auc_score(y_true, probs, average='micro')
logs['auc'] = score
Hence next callback which I wrote was to achieve the target.
class scoreTarget(Callback):
def __init__(self, target):
super(scoreTarget, self).__init__()
self.target = target
def on_epoch_end(self, epoch, logs={}):
acc = logs['auc']
if acc >= self.target:
self.model.stop_training = True
The list of call back used is below:
roc_callback = rocback((X_test_pooled_output, y_test))
early_stopping = EarlyStopping(patience=5)
tensorboard = TensorBoard()
reduce_lr = ReduceLROnPlateau(patience=3)
target = scoreTarget(0.90)
callbacks = [
roc_callback,
early_stopping,
tensorboard,
reduce_lr,
target,
]
The classifier function which I wrote is below:
class ReviewClassifier(Model):
def __init__(self):
super(ReviewClassifier, self).__init__()
self.dense_1 = Dense(64, activation='relu')
self.dense_2 = Dense(32, activation='relu')
self.dense_3 = Dense(16, activation='relu')
self.classify = Dense(1, activation='sigmoid')
self.dropout_1 = Dropout(0.2)
self.dropout_2 = Dropout(0.2)
self.dropout_3 = Dropout(0.2)
def call(self, inputs):
x = self.dense_1(inputs)
x = self.dropout_1(x)
x = self.dense_2(x)
x = self.dropout_2(x)
x = self.dense_3(x)
x = self.dropout_3(x)
x = self.classify(x)
return x
review_classifier = ReviewClassifier()
review_classifier.build((None, 768))
review_classifier.summary()
The compile funtcion I wrote is this:
review_classifier.compile(loss='binary_crossentropy',
optimizer='adam',metrics=[rocback])
fit function:
!rm -rf ./logs/*
history = review_classifier.fit(X_train_pooled_output, y_train,
batch_size=32, epochs=100,
callbacks=callbacks,
validation_data=(X_test_pooled_output, y_test))
The error received is:
Epoch 1/100
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-54-fd7595a2c88c> in <module>()
1 get_ipython().system('rm -rf ./logs/*')
----> 2 history = review_classifier.fit(X_train_pooled_output, y_train, batch_size=32, epochs=100,callbacks=callbacks, validation_data=(X_test_pooled_output, y_test))
9 frames
/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/func_graph.py in wrapper(*args, **kwargs)
975 except Exception as e: # pylint:disable=broad-except
976 if hasattr(e, "ag_error_metadata"):
--> 977 raise e.ag_error_metadata.to_exception(e)
978 else:
979 raise
TypeError: in user code:
/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py:805 train_function *
return step_function(self, iterator)
/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py:795 step_function **
outputs = model.distribute_strategy.run(run_step, args=(data,))
/usr/local/lib/python3.7/dist-packages/tensorflow/python/distribute/distribute_lib.py:1259 run
return self._extended.call_for_each_replica(fn, args=args, kwargs=kwargs)
/usr/local/lib/python3.7/dist-packages/tensorflow/python/distribute/distribute_lib.py:2730 call_for_each_replica
return self._call_for_each_replica(fn, args, kwargs)
/usr/local/lib/python3.7/dist-packages/tensorflow/python/distribute/distribute_lib.py:3417 _call_for_each_replica
return fn(*args, **kwargs)
/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py:788 run_step **
outputs = model.train_step(data)
/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py:758 train_step
self.compiled_metrics.update_state(y, y_pred, sample_weight)
/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/compile_utils.py:408 update_state
metric_obj.update_state(y_t, y_p, sample_weight=mask)
/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/utils/metrics_utils.py:90 decorated
update_op = update_state_fn(*args, **kwargs)
/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/metrics.py:177 update_state_fn
return ag_update_state(*args, **kwargs)
/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/metrics.py:618 update_state **
matches = ag_fn(y_true, y_pred, **self._fn_kwargs)
TypeError: __init__() takes 2 positional arguments but 3 were given
Any idea what am I doing wrong in the custom callback which I created for roc?
Let me know what more inputs you need.
As we've mentioned in the comment, you can use built-in tf.keras.metrics.AUC while compiling the model, easy-fast-efficient. But also no problem to use either sklearn.metrics.roc_auc_score. But using it in callback may slow down your training time. Try this:
class ROAUCMetrics(tf.keras.callbacks.Callback):
def __init__(self, val_data):
super().__init__()
self.valid_x = val_data[0]
self.valid_y = val_data[1]
def on_train_begin(self, logs={}):
self.val_aucs = []
def on_epoch_end(self, epoch, logs={}):
pred = self.model.predict(self.valid_x)
val_auc = roc_auc_score(self.valid_y, pred, average='micro')
print('\nval-roc-auc: %s' % (str(round(val_auc,4))),end=100*' '+'\n')
self.val_aucs.append(val_auc)
return
# sklearn auc
roc = ROAUCMetrics(val_data=(x_val, y_val))
# tf.keras auc
model.compile(.., ..., metrics=["AUC"])
# running
model.fit(x_train, y_train, batch_size=1024,
epochs=..., callbacks=[roc],
validation_data=(x_val, y_val))
# get the values of auc, computed using sklearn auc
roc.val_aucs
However, note that both compute AUC differently, one uses Approximate AUC, another one uses Riemann sum, I've tested in one example and they pretty comparable but sometimes didn't.
Based on your 1st comment, your set up should look like this:
# (1)
# compile with no metrics - as we have custom callback metric to use
review_classifier.compile(loss='binary_crossentropy',optimizer='adam')
# (2)
# or,
# we can add another metrics e.g 'accuracy' or whatever
# here we use built-in AUC
review_classifier.compile(loss='binary_crossentropy',
optimizer='adam',metrics=['AUC'])
# sklearn auc
roc = ROAUCMetrics(val_data=(x_val, y_val))
early_stopping = EarlyStopping(patience=5)
tensorboard = TensorBoard()
reduce_lr = ReduceLROnPlateau(patience=3)
target = scoreTarget(0.90)
callbacks = [
roc,
early_stopping,
tensorboard,
reduce_lr,
target,
]
# fitting
model.fit(x_train, y_train, batch_size=1024,
epochs=..., callbacks=callbacks,
validation_data=(x_val, y_val))
I've a simple model and need access of intermediate layers within a custom callback to get intermediate predictions.
import tensorflow as tf
import numpy as np
X = np.ones((8,16))
y = np.sum(X, axis=1)
class CustomCallback(tf.keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs=None):
get_output = tf.keras.backend.function(
inputs = self.model.layers[0].input,
outputs = self.model.layers[1].output
)
print("\nLayer output: ", get_output(X))
If I build the model by sub-classing like below:
class Model(tf.keras.Model):
def build(self, input_shape):
self.dense1 = tf.keras.layers.Dense(units=32)
self.dense2 = tf.keras.layers.Dense(units=1)
def call(self, input_tensor):
x = self.dense1(input_tensor)
x = self.dense2(x)
return x
model = Model()
model.compile(optimizer='adam',loss='mean_squared_error', metrics='accuracy')
model.fit(X,y, epochs=2, callbacks=[CustomCallback()])
I get following error:
<ipython-input-8-bab75191182e> in on_epoch_end(self, epoch, logs)
2 def on_epoch_end(self, epoch, logs=None):
3 get_output = tf.keras.backend.function(
----> 4 inputs = self.model.layers[0].input,
5 outputs = self.model.layers[1].output
6 )
.
.
AttributeError: Layer dense is not connected, no input to return.
whereas if I build model using functional API like below, it works as expected.
initial = tf.keras.layers.Input((16,))
x = tf.keras.layers.Dense(units=32)(initial)
final = tf.keras.layers.Dense(units=1)(x)
model = tf.keras.Model(initial, final)
model.compile(optimizer='adam',loss='mean_squared_error', metrics='accuracy')
model.fit(X,y, epochs=2, callbacks=[CustomCallback()])
What's causing the error in case of model sub-classing?
TF version: 2.2.0
I would like to change optimizer in the middle of training.
I use the code from this post:
Changing optimizer in keras during training
as follows:
model = Model(inputs=inputs, outputs=conv12)
def rmse(y_true, y_pred):
return backend.sqrt(backend.mean(backend.square(y_pred - y_true)))#, axis=-1))
class OptimizerChanger(EarlyStopping):
def __init__(self, on_train_end, **kwargs):
self.do_on_train_end = on_train_end
super(OptimizerChanger, self).__init__(**kwargs)
def on_train_end(self, logs=None):
super(OptimizerChanger, self).on_train_end(self, logs)
self.do_on_train_end()
def do_after_training():
model.compile(optimizer='sgd', loss='mean_squared_error', metrics=['mae', rmse])
model.fit(x_train_n, y_train_n, batch_size=10, epochs=200, validation_split=0.05, shuffle=True)
changer = OptimizerChanger(on_train_end= do_after_training,
monitor='val_rmse',
min_delta=5,
patience=10)
model.compile(loss='mean_squared_error',
optimizer='adam',
metrics=['mae', rmse])
history = model.fit(x_train_n, y_train_n, batch_size=10, epochs=200, validation_split=0.05, shuffle=True, callbacks=[changer])
I get the following error:
super(OptimizerChanger, self).on_train_end(self, logs)
TypeError: on_train_end() takes from 1 to 2 positional arguments but 3 were given
what is the third parameter that is being passed? is it implicit? how do I call it?
You're calling the method from an instance and you are passing self to it. So it's redundant.
super(OptimizerChanger, self).on_train_end(self, logs)
This line should be
super(OptimizerChanger, self).on_train_end(logs)