I have a multi output(200) binary classification model which I wrote in keras.
In this model I want to add additional metrics such as ROC and AUC but to my knowledge keras dosen't have in-built ROC and AUC metric functions.
I tried to import ROC, AUC functions from scikit-learn
from sklearn.metrics import roc_curve, auc
from keras.models import Sequential
from keras.layers import Dense
.
.
.
model.add(Dense(200, activation='relu'))
model.add(Dense(300, activation='relu'))
model.add(Dense(400, activation='relu'))
model.add(Dense(300, activation='relu'))
model.add(Dense(200,init='normal', activation='softmax')) #outputlayer
model.compile(loss='categorical_crossentropy', optimizer='adam',metrics=['accuracy','roc_curve','auc'])
but it's giving this error:
Exception: Invalid metric: roc_curve
How should I add ROC, AUC to keras?
Due to that you can't calculate ROC&AUC by mini-batches, you can only calculate it on the end of one epoch. There is a solution from jamartinh, I patch the code below for convenience:
from sklearn.metrics import roc_auc_score
from keras.callbacks import Callback
class RocCallback(Callback):
def __init__(self,training_data,validation_data):
self.x = training_data[0]
self.y = training_data[1]
self.x_val = validation_data[0]
self.y_val = validation_data[1]
def on_train_begin(self, logs={}):
return
def on_train_end(self, logs={}):
return
def on_epoch_begin(self, epoch, logs={}):
return
def on_epoch_end(self, epoch, logs={}):
y_pred_train = self.model.predict_proba(self.x)
roc_train = roc_auc_score(self.y, y_pred_train)
y_pred_val = self.model.predict_proba(self.x_val)
roc_val = roc_auc_score(self.y_val, y_pred_val)
print('\rroc-auc_train: %s - roc-auc_val: %s' % (str(round(roc_train,4)),str(round(roc_val,4))),end=100*' '+'\n')
return
def on_batch_begin(self, batch, logs={}):
return
def on_batch_end(self, batch, logs={}):
return
roc = RocCallback(training_data=(X_train, y_train),
validation_data=(X_test, y_test))
model.fit(X_train, y_train,
validation_data=(X_test, y_test),
callbacks=[roc])
A more hackable way using tf.contrib.metrics.streaming_auc:
import numpy as np
import tensorflow as tf
from sklearn.metrics import roc_auc_score
from sklearn.datasets import make_classification
from keras.models import Sequential
from keras.layers import Dense
from keras.utils import np_utils
from keras.callbacks import Callback, EarlyStopping
# define roc_callback, inspired by https://github.com/keras-team/keras/issues/6050#issuecomment-329996505
def auc_roc(y_true, y_pred):
# any tensorflow metric
value, update_op = tf.contrib.metrics.streaming_auc(y_pred, y_true)
# find all variables created for this metric
metric_vars = [i for i in tf.local_variables() if 'auc_roc' in i.name.split('/')[1]]
# Add metric variables to GLOBAL_VARIABLES collection.
# They will be initialized for new session.
for v in metric_vars:
tf.add_to_collection(tf.GraphKeys.GLOBAL_VARIABLES, v)
# force to update metric values
with tf.control_dependencies([update_op]):
value = tf.identity(value)
return value
# generation a small dataset
N_all = 10000
N_tr = int(0.7 * N_all)
N_te = N_all - N_tr
X, y = make_classification(n_samples=N_all, n_features=20, n_classes=2)
y = np_utils.to_categorical(y, num_classes=2)
X_train, X_valid = X[:N_tr, :], X[N_tr:, :]
y_train, y_valid = y[:N_tr, :], y[N_tr:, :]
# model & train
model = Sequential()
model.add(Dense(2, activation="softmax", input_shape=(X.shape[1],)))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy', auc_roc])
my_callbacks = [EarlyStopping(monitor='auc_roc', patience=300, verbose=1, mode='max')]
model.fit(X, y,
validation_split=0.3,
shuffle=True,
batch_size=32, nb_epoch=5, verbose=1,
callbacks=my_callbacks)
# # or use independent valid set
# model.fit(X_train, y_train,
# validation_data=(X_valid, y_valid),
# batch_size=32, nb_epoch=5, verbose=1,
# callbacks=my_callbacks)
Like you, I prefer using scikit-learn's built in methods to evaluate AUROC. I find that the best and easiest way to do this in keras is to create a custom metric. If tensorflow is your backend, implementing this can be done in very few lines of code:
import tensorflow as tf
from sklearn.metrics import roc_auc_score
def auroc(y_true, y_pred):
return tf.py_func(roc_auc_score, (y_true, y_pred), tf.double)
# Build Model...
model.compile(loss='categorical_crossentropy', optimizer='adam',metrics=['accuracy', auroc])
Creating a custom Callback as mentioned in other answers will not work for your case since your model has multiple ouputs, but this will work. Additionally, this methods allows the metric to be evaluated on both training and validation data whereas a keras callback does not have access to the training data and can thus only be used to evaluate performance on the training data.
The following solution worked for me:
import tensorflow as tf
from keras import backend as K
def auc(y_true, y_pred):
auc = tf.metrics.auc(y_true, y_pred)[1]
K.get_session().run(tf.local_variables_initializer())
return auc
model.compile(loss="binary_crossentropy", optimizer='adam', metrics=[auc])
I solved my problem this way
consider you have testing dataset x_test for features and y_test for its corresponding targets.
first we predict targets from feature using our trained model
y_pred = model.predict_proba(x_test)
then from sklearn we import roc_auc_score function and then simple pass the original targets and predicted targets to the function.
roc_auc_score(y_test, y_pred)
You can monitor auc during training by providing metrics the following way:
METRICS = [
keras.metrics.TruePositives(name='tp'),
keras.metrics.FalsePositives(name='fp'),
keras.metrics.TrueNegatives(name='tn'),
keras.metrics.FalseNegatives(name='fn'),
keras.metrics.BinaryAccuracy(name='accuracy'),
keras.metrics.Precision(name='precision'),
keras.metrics.Recall(name='recall'),
keras.metrics.AUC(name='auc'),
]
model = keras.Sequential([
keras.layers.Dense(16, activation='relu', input_shape=(train_features.shape[-1],)),
keras.layers.Dense(1, activation='sigmoid'),
])
model.compile(
optimizer=keras.optimizers.Adam(lr=1e-3)
loss=keras.losses.BinaryCrossentropy(),
metrics=METRICS)
for a more detailed tutorial see:
https://www.tensorflow.org/tutorials/structured_data/imbalanced_data
'roc_curve','auc' are not standard metrics you can't pass them like that to metrics variable, this is not allowed.
You can pass something like 'fmeasure' which is a standard metric.
Review the available metrics here: https://keras.io/metrics/
You may also want to have a look at making your own custom metric: https://keras.io/metrics/#custom-metrics
Also have a look at generate_results method mentioned in this blog for ROC, AUC...
https://vkolachalama.blogspot.in/2016/05/keras-implementation-of-mlp-neural.html
Adding to above answers, I got the error "ValueError: bad input shape ...", so I specify the vector of probabilities as follows:
y_pred = model.predict_proba(x_test)[:,1]
auc = roc_auc_score(y_test, y_pred)
print(auc)
Set your model architecture with tf.keras.metrics.AUC():
Read the Keras documentation on Classification metrics based on True/False positives & negatives.
def model_architecture_ann(in_dim,lr=0.0001):
model = Sequential()
model.add(Dense(512, input_dim=X_train_filtered.shape[1], activation='relu'))
model.add(Dense(1, activation='sigmoid'))
opt = keras.optimizers.SGD(learning_rate=0.001)
auc=tf.keras.metrics.AUC()
model.compile(loss='binary_crossentropy', optimizer=opt, metrics=[tf.keras.metrics.AUC(name='auc')])
model.summary()
return model
Related
so I used deeplearning to improve my model accuracy, but when I check with bayesian classifier I got 91.67% accuracy
then I check with deep learning, but it doesn't improve max I get 91.67%
I have to improve my accuracy, I want to try using Tuning, but I don't know how
My dataset has 3 class
So please help me, at least I get 92% accuracy
import pandas
from keras.models import Sequential
from keras.layers import Dense
from keras.wrappers.scikit_learn import KerasClassifier
from keras.utils import np_utils
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import KFold
from sklearn.preprocessing import LabelEncoder
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split
# load dataset
dataframe = pandas.read_csv("pca_aug.csv", header=None)
dataset = dataframe.values
X = dataset[:,0:300].astype(float)
Y = dataset[:,300]
xtrain,xtest,ytrain,ytest= train_test_split(X,Y,test_size=0.4,random_state=0)
# encode class values as integers
def konversi(Y):
encoder = LabelEncoder()
encoder.fit(Y)
encoded_Y = encoder.transform(Y)
# convert integers to dummy variables (i.e. one hot encoded)
dummy_y = np_utils.to_categorical(encoded_Y)
return dummy_y
ytrain_dummy= konversi(ytrain)
ytest_dummy= konversi(ytest)
# create model
model = Sequential()
model.add(Dense(1000, input_dim=300, activation='relu'))
model.add(Dense(3, activation='softmax'))
# Compile model
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
nepochs = 20
nbatch = 5
model.fit(xtrain, ytrain_dummy, epochs=nepochs, batch_size=nbatch)
_, accuracy = model.evaluate(xtest, ytest_dummy)
print('Accuracy: %.2f' % (accuracy*100))
You can't set a limit on the accuracy, before training the model, it will introduce biases to the results. The performance metric depends on the data, noise present in the data, training, and the model.
You can use a hyperparameter searching library like keras-tuner.
import kerastuner as kt
from tensorflow import keras
def build_model(hp):
...
return model
tuner = kt.RandomSearch(
build_model,
objective='val_loss',
max_trials=5)
tuner.search(x_train, y_train, epochs=5, validation_data=(x_val, y_val))
best_model = tuner.get_best_models()[0]
I am testing the code below.
#%matplotlib inline
import seaborn as sns
import pandas as pd
import numpy as np
from sklearn.model_selection import cross_validate
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegressionCV
iris = sns.load_dataset("iris")
iris.head()
sns.pairplot(iris, hue='species')
X = iris.values[:, 0:4]
y = iris.values[:, 4]
train_X, test_X, train_y, test_y = train_test_split(X, y, train_size=0.5, random_state=0)
lr = LogisticRegressionCV()
lr.fit(train_X, train_y)
pred_y = lr.predict(test_X)
print("Test fraction correct (Accuracy) = {:.2f}".format(lr.score(test_X, test_y)))
# Test fraction correct (Accuracy) = 0.93
import keras
from keras.models import Sequential
from keras.layers.core import Dense, Activation
from keras.utils import np_utils
train_y_ohe = pd.get_dummies(train_y)
test_y_ohe = pd.get_dummies(test_y)
model = Sequential()
model.add(Dense(16, input_shape=(4,)))
model.add(Activation('sigmoid'))
model.add(Dense(3))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam')
loss, accuracy = model.evaluate(test_X, test_y_ohe, show_accuracy=True, verbose=0)
print("Test fraction correct (Accuracy) = {:.2f}".format(accuracy))
Everything works fine until the next-to-last-line of code.
When I try to run this:
loss, accuracy = model.evaluate(test_X, test_y_ohe, show_accuracy=True, verbose=0)
I get this error:
TypeError: evaluate() got an unexpected keyword argument 'show_accuracy'
I did a bit of research, and found that 'show_accuracy=True' may have been depreciated a short time ago. Is there some other way of doing this now? How can I evaluate, and print, the accuracy of the model?
I found the code sample here:
https://blog.fastforwardlabs.com/2016/02/24/hello-world-in-keras-or-scikit-learn-versus.html
The show_accuracy argument is deprecated in new versions of keras,remove this argument from model.evaluate() and use instead metrics=['accuracy'] in model.compile()
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
# fit model
train_y_ohe = pd.get_dummies(train_y)
model.fit(train_X, train_y_ohe,epochs=1000,batch_size=20)
loss, accuracy = model.evaluate(test_X, test_y_ohe, verbose=0)
print("Test fraction correct (Accuracy) = {:.2f}".format(accuracy))
#Test fraction correct (Accuracy) = 0.97
I'm new to Python (although not to programming - I'm usually programming in JavaScript) and I'm very interested in AI development.
Recently I've been trying to develop a deep learning algorithm by following this article.
My goal is to predict a set of 7 numbers, based on a CSV file that contains a large list, with each row having 7 numbers as well. The order of the list matters.
I ended up having the following code:
from keras.models import Sequential
from keras.layers import Dense
from sklearn.model_selection import train_test_split
from numpy import loadtxt, random
random.seed(seed)
dataset = loadtxt("data/Lotto.csv", delimiter=",", skiprows=1)
X = dataset[:, 0:7]
Y = dataset[:, 6]
(X_train, X_test, Y_train, Y_test) = train_test_split(X, Y, test_size=0.33, random_state=4)
model = Sequential()
model.add(Dense(8, input_dim=7, kernel_initializer="uniform", activation="relu"))
model.add(Dense(6, kernel_initializer="uniform", activation="relu"))
model.add(Dense(1, kernel_initializer="uniform", activation="sigmoid"))
model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])
model.fit(X_train, Y_train, validation_data=(X_test, Y_test), epochs=100, batch_size=5, shuffle=False)
scores = model.evaluate(X_test, Y_test)
print("Accuracy: %.2f%%" %(scores[1] * 100))
After running it in Google Colaboratory, while I'm not getting any errors - I noticed that for each epoch, the loss result doesn't change, and as a result, I keep getting low accuracy (~6%).
What am I doing wrong?
Try changing optimizer to RMSprop with learning rate of around 0.0001.
RMSprop is usually better than most optimizers and gives better accuracy and lesser loss than others. You could alternatively try SGD, which is also a good optimizer.
Also increase number of parameters, as more trainable parameters leads to the model being trained with more precision and gives a much accurate prediction.
You could update the code to tensorflow 2.x and change the code to :
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense
from sklearn.model_selection import train_test_split
from numpy import loadtxt, random
#Rest of the code
.......
.......
.......
model = Sequential()
model.add(Dense(64, input_shape=(7,), activation='relu', kernel_initializer='uniform'))
model.add(Dense(64, kernel_initializer="uniform", activation="relu"))
model.add(Dense(1, kernel_initializer="uniform", activation="sigmoid"))
model.compile(loss="binary_crossentropy", optimizer=tf.keras.optimizers.RMSprop(learning_rate=0.0001), metrics=["accuracy"])
model.fit(X_train, Y_train, validation_data=(X_test, Y_test), epochs=100, batch_size=5, shuffle=False)
scores = model.evaluate(X_test, Y_test)
print("Accuracy: %.2f%%" %(scores[1] * 100))
Correct me if I'm wrong, but by the looks of it your input is a list of 7 numbers, and you want to output the 7th number in that list. Now by using a sigmoid activation in your last layer, you're restricting your model output to the interval (0,1). Are you sure that your data is in this interval?
Also, your model is way to complicated for that task. You really need only one dense layer without an activation or bias to be able to do this.
I am training a binary classifier using Keras.
model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=[auroc,'accuracy'])
I use a custom metric, AUROC.as in here
import tensorflow as tf
from sklearn.metrics import roc_auc_score
def auroc(y_true, y_pred):
return tf.py_func(roc_auc_score, (y_true, y_pred), tf.double)
So far, I have encoded my target as one-hot encoding
and I had a last layer as
from keras.utils import to_categorical
y = to_categorical(y)
[...]
model.add(Dense(2, activation='sigmoid'))
I have learned that in principle here Keras binary_crossentropy vs categorical_crossentropy performance?I should not perform categorical encoding, and I should predict only one class using
# y = to_categorical(y)
[...]
model.add(Dense(1, activation='sigmoid'))
However, if I apply this and only this change my training auroc changes dramatically, from high 0.90s to 0.50. Even more strangely, val_auroc loss seems unaffected
How did that happen?
I am trying to solve a regression with Keras but MSE is huge, I mean like 29346217.6819
I am really new, so do you have any suggestions to make the model give reasonable mse? I am not sure even my data is OK or problematic but those are actual sales data.
Data (about to 3000 lines. I use 2000 for training and 1000 for testing)
Full data is here
ProductNo,Day,Month,CartonSales
1,6,02,2374
1,3,02,2374
1,6,04,2374
1,6,04,2374
1,3,06,2374
1,6,09,2374
1,1,09,2374
1,6,09,2374
1,6,10,2374
Code
from keras import optimizers
from keras.callbacks import Callback
from numpy import array
from keras.models import Sequential
from keras.layers import Dense, Dropout
from matplotlib import pyplot
import pandas as pds
# prepare sequence
class TestCallback(Callback):
def __init__(self, test_data):
self.test_data = test_data
def on_epoch_end(self, epoch, logs={}):
x, y = self.test_data
loss, acc = self.model.evaluate(x, y, verbose=0)
print('\nTesting loss: {}, acc: {}\n'.format(loss, acc))
dataframe = pds.read_csv('pmidata.csv', usecols=[0, 1, 2, 3])
dataframe = dataframe.sample(frac=1)
dataframeX_train = dataframe.iloc[0:2000][['ProductNo', 'Day', 'Month']]
dataframeY_train = dataframe.iloc[0:2000][['CartonSales']]
dataframeX_test = dataframe.iloc[2001:3001][['ProductNo', 'Day', 'Month']]
dataframeY_test = dataframe.iloc[2001:3001][['CartonSales']]
# create model
model = Sequential()
model.add(Dense(3, input_dim=3, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(1))
model.compile(loss='mse', optimizer='adam', metrics=['mse'])
#sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
#model.compile(loss='mse', optimizer=sgd, metrics=['mse'])
# train model
#history = model.fit(dataframe, dataframe, epochs=500, batch_size=len(X), verbose=2)
history = model.fit(dataframeX_train, dataframeY_train, epochs=100, batch_size=4, verbose=2, callbacks=[TestCallback((dataframeX_test, dataframeY_test))])
# plot metrics
pyplot.plot(history.history['mean_squared_error'])
pyplot.show()
As far as i can tell from your code above, your y values are CartonSales. Sales can have large values and large range and that's probably why you get such a high error. You could use mean_squared_logarithmic_error instead of mean square error but i would suggest to do the following.
Continue using mean square error.
log transform you y values and later exp transform you predictions
import numpy as np
dataframeY_train = np.log(dataframeY_train)
dataframeY_test = np.log(dataframeY_test )
....
predictions=model.predict(dataframeX_test)[:,0]
predictions = np.exp(predictions)