We Keep Coding

X has 14 features, but RandomForestClassifier is expecting 20 features as input

Could I please ask, I have this code (a reproducible example):
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import StratifiedKFold,KFold
from sklearn.feature_selection import SelectKBest
#from xgboost import XGBClassifier
from sklearn.feature_selection import mutual_info_classif
from sklearn.feature_selection import SelectKBest, RFECV
from sklearn.pipeline import Pipeline
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import make_scorer, recall_score, accuracy_score, precision_score
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import make_scorer
from sklearn.metrics import precision_score,recall_score,f1_score,roc_auc_score
from sklearn import metrics
from sklearn.datasets import make_classification
from numpy import mean
from sklearn.model_selection import train_test_split
from numpy import std
from sklearn.utils import shuffle
import numpy as np
from sklearn.metrics import roc_curve
from sklearn.pipeline import Pipeline
import matplotlib.pyplot as plt
import pickle
#import neptune.new as neptune
import pandas as pd
import shap
full_X_train,full_y_train = make_classification(n_samples =500,n_features = 20, random_state=1, n_informative=10,n_redundant=10)
def run_model_with_grid_search(param_grid={}, output_plt_file = 'plt.png',model_name=RandomForestClassifier(),X_train=full_X_train,y_train=full_y_train,model_id='random_forest_with_hpo_no_fs_geno_class', n_splits=5, output_file='random_forest_with_hpo_no_fs_geno_class.txt'):
cv_outer = KFold(n_splits=5,shuffle=True,random_state=1)
acc_list = list()
f1_list = list()
precision_list = list()
recall_list = list()
auc_list = list()
#for ROC curve
tprs = []
base_fpr = np.linspace(0, 1, 101)
plt.figure(figsize=(5, 5))
plt.axes().set_aspect('equal', 'datalim')
count = 0
list_shap_values = list()
list_test_sets = list()
for train_ix,test_ix in cv_outer.split(X_train):
split_x_train, split_x_test = X_train[train_ix,:],X_train[test_ix,:] #add in .iloc
split_y_train, split_y_test = y_train[train_ix],y_train[test_ix] #add in .iloc
cv_inner = KFold(n_splits=3,shuffle=True,random_state=1)
model = model_name
rfecv = RFECV(estimator=model, step=1,scoring='accuracy',cv=cv_inner) #change scoring to scoring='roc_auc'
pipeline = Pipeline([('feature_sele',rfecv),('clf_cv',model)])
search = GridSearchCV(pipeline,param_grid=param_grid,scoring='roc_auc',cv=cv_inner,refit=True).fit(split_x_train,split_y_train)
best_model = search.best_estimator_
best_model_shap = search.best_estimator_['clf_cv'].fit(split_x_train,split_y_train)
print(search.best_params_)
yhat = best_model.predict(split_x_test) #changed from best_model and split_x_test
accuracy = accuracy_score(split_y_test,yhat)
acc_list.append(accuracy)
f1_sc = f1_score(split_y_test,yhat)
f1_list.append(f1_sc)
precision_sc = precision_score(split_y_test,yhat)
precision_list.append(precision_sc)
recall_sc = recall_score(split_y_test,yhat)
recall_list.append(recall_sc)
fpr, tpr, _ = roc_curve(split_y_test, yhat)
auc = metrics.auc(fpr,tpr)
auc_list.append(auc)
plt.plot(fpr, tpr, 'b', alpha=0.15)
tpr = np.interp(base_fpr, fpr, tpr)
tpr[0] = 0.0
tprs.append(tpr)
count +=1
print('>acc=%.3f,est=%.3f,cfg=%s'%(accuracy,search.best_score_,search.best_params_))
explainer = shap.TreeExplainer(best_model_shap)
shap_values = explainer.shap_values(split_x_test)
list_shap_values.append(shap_values)
list_test_sets.append(test_ix)
test_set = list_test_sets[0]
shap_values = np.array(list_shap_values[0])
for i in range(1,len(list_test_sets)):
test_set = np.concatenate((test_set,list_test_sets[i]),axis=0)
shap_values = np.concatenate((shap_values,np.array(list_shap_values[i])),axis=1)
X_test_df = pd.DataFrame(full_X_train[test_set])
cols = X_test_df.columns
shap_sum = np.abs(shap_values[1,:,:]).mean(0)
importance_df = pd.DataFrame({
'column_name':cols,
'shap_values':shap_sum
})
importance_df.sort_values('shap_values',ascending=False)
print('Accuracy: %.3f (%.3f)' % (mean(acc_list),std(acc_list)))
print('F1: %.3f (%.3f)' % (mean(f1_list),std(f1_list)))
print('Precision: %.3f (%.3f)' % (mean(precision_list),std(precision_list)))
print('Recall: %.3f (%.3f)' % (mean(recall_list),std(recall_list)))
print('AUC: %.3f (%.3f)' % (mean(auc_list),std(auc_list)))
tprs = np.array(tprs)
mean_tprs = tprs.mean(axis=0)
tpr_std = tprs.std(axis=0)
tprs_upper = np.minimum(mean_tprs + tpr_std, 1)
tprs_lower = mean_tprs - tpr_std
plt.plot(base_fpr, mean_tprs, 'b')
plt.fill_between(base_fpr, tprs_lower, tprs_upper, color='grey', alpha=0.3)
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([-0.01, 1.01])
plt.ylim([-0.01, 1.01])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.title('ROC for stratified 5-fold CV (blue line = mean)')
plt.savefig(output_plt_file)
print(importance_df)
return
param_grid = [{
'clf_cv__min_samples_leaf':[1,3,5],
[200,500,700,1000,1500,2000]
}]
run_model_with_grid_search(param_grid=param_grid)
I get the error:
X has 14 features, but RandomForestClassifier is expecting 20 features as input
I can see other people have had this issue understand the problem (I think) - it's because I do feature selection before I build a model, but then I don't transform my X test data to be the same dimensions.
I was trying to implement solutions based on seeing other people's work e.g.
search = GridSearchCV(pipeline,param_grid=param_grid,scoring='roc_auc',cv=cv_inner,refit=True).fit(split_x_train,split_y_train)
best_model = search.best_estimator_
best_model_shap = search.best_estimator_['clf_cv'].fit_transform(split_x_train,split_y_train)
print(search.best_params_)
transformed_x_test = best_model.transform(split_x_test)
yhat = best_model.predict(transformed_x_test) #changed from best_model and split_x_test
But that is leading to other errors (e.g. in this case, 'GridSerchCV object has no attribute fit_transform') - so I'm just not clear how to implement other's solutions to my issue.
Could someone please demonstrate to me how to get this piece of code working (it should all work except for this error anyway). Also, if someone could show me how to print which features are being selected by the feature selection process that would be great but I think I can figure that out after this starts working.

How to compute false positive rate of an imbalanced dataset for Stratified K fold cross validation?

The below lines are the sample code where I am able to compute accuracy, precision, recall, and f1 score. How can I also compute a false positive rate (FPR) for Stratified K fold cross-validation?
from sklearn.metrics import make_scorer, accuracy_score, precision_score, recall_score,
f1_score
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import cross_validate
scoring = {'accuracy' : make_scorer(accuracy_score),
'precision' : make_scorer(precision_score),
'recall' : make_scorer(recall_score),
'f1_score' : make_scorer(f1_score)}
skfold = StratifiedKFold(n_splits=10)
dt_clf = DecisionTreeClassifier()
results = cross_validate(estimator=dt_clf,
X=data_train_X,
y=target_train_Y,
cv=skfold,
scoring=scoring)
print("Results", results)

You could define a custom scorer as follows:
from sklearn.metrics import make_scorer, accuracy_score, precision_score, recall_score, f1_score
from sklearn.model_selection import StratifiedKFold, cross_validate
from sklearn.tree import DecisionTreeClassifier
from sklearn.datasets import make_classification
def false_positive_rate(y_true, y_pred):
# false positive
fp = ((y_pred == 1) & (y_true == 0)).sum()
# true negative
tn = ((y_pred == 0) & (y_true == 0)).sum()
# false positive rate
return fp / (fp + tn)
scoring = {
'accuracy': make_scorer(accuracy_score),
'precision': make_scorer(precision_score),
'recall': make_scorer(recall_score),
'f1_score': make_scorer(f1_score),
'false_positive_rate': make_scorer(false_positive_rate),
}
skf = StratifiedKFold(n_splits=3)
clf = DecisionTreeClassifier(random_state=42)
X, y = make_classification(random_state=42)
results = cross_validate(estimator=clf, X=X, y=y, cv=skf, scoring=scoring)
print(results['test_false_positive_rate'])
# [0.11764706 0.11764706 0.0625]

I wrote this code with Logistic Regression. You can substitute it with any other binary classification algorithm you'd like.
#Importing required libraries
from sklearn.model_selection import KFold
from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
data = load_breast_cancer(as_frame = True)
df = result.frame
X = result.iloc[:,:-1]
y = result.iloc[:,-1]
#shffling
X = X.sample(frac = 1)
y= y.sample(frac = 1)
#Implementing cross validation
kf = KFold(n_splits=10)#, random_state=None
model = LogisticRegression(max_iter=1000000)#(solver= 'liblinear')
acc_score = list()
res_tpr = list()
res_fpr = list()
for train_index , test_index in kf.split(X):
#X_train , X_test = X.iloc[train_index,:],X.iloc[test_index,:]
#y_train , y_test = y[train_index] , y[test_index]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
model.fit(X_train,y_train)
pred_values = model.predict(X_test)
tn, fp, fn, tp = confusion_matrix(y_test, pred_values, labels=[0, 1]).ravel()
print(f'True Positives: {tp}')
print(f'False Positives: {fp}')
print(f'True Negatives: {tn}')
print(f'False Negatives: {fn}')
tpr=(np.divide(tp,(tp+fn)))
fpr=(np.divide(fp,(fp+tn)))
if tp==0:
tpr=0
if fp==0:
fpr=0
print('tpr=%.4f fpr=%.3f' % ( mean(tpr), mean(fpr)))
res_tpr.append(mean(tpr))
res_fpr.append(mean(fpr))
print('---------------------')
acc = accuracy_score(pred_values , y_test)
acc_score.append(acc)
avg_acc_score = np.sum(acc_score)/10
total_tpr=np.sum(res_tpr)/10
total_fpr=np.sum(res_fpr)/10
print('\n\n',' total_tpr=%.4f total_fpr=%.3f' % (total_tpr,total_fpr))
#print('\n\n','accuracy of each fold - {}'.format(acc_score))
print('\n\n','Avg accuracy : {}'.format(avg_acc_score))

AttributeError: 'numpy.ndarray' object has no attribute 'score' error

I have tried to look for a problem but there is nothing Im seeing wrong here. What could it be? This is for trying binary classification in SVM for the fashion MNIST data set but only classifying 5 and 7.
import pandas as pd
import numpy as np
import seaborn as sns
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn import svm
from sklearn.preprocessing import MinMaxScaler
from sklearn.svm import SVR
from sklearn.model_selection import KFold
import matplotlib.pyplot as plt
trainset = 'mnist_train.xlsx'
trs = pd.read_excel(trainset)
testset = 'mnist_test.xlsx'
tes = pd.read_excel(testset)
xtrain = trs.iloc[:, [1, 783]]
ytrain = trs.iloc[:, 0]
xtest = tes.iloc[:, [1, 783]]
ytest = tes.iloc[:, 0]
##Linear SVC
svclassifier = SVC(kernel='linear', C=1)
svclassifier.fit(xtest, ytest)
ypred = svclassifier.predict(xtest)
print(ypred.score(xtrain, ytrain))
print(ypred.score(xtest, ytest))
##Gaussian SVC
svclassifier = SVC(kernel='rbf', C=1)
svclassifier.fit(xtrain, ytrain)
ypred = svclassifier.predict(xtest)
print(ypred.score(xtrain, ytrain))
print(ypred.score(xtest, ytest))

ypred is an array of predicted class labels, so the exception makes sense.
What you should do is use the classifier’s score method:
svclassifier = SVC(kernel='rbf', C=1)
svclassifier.fit(xtrain, ytrain)
# ypred = svclassifier.predict(xtest) # We don’t actually use this.
print(svclassifier.score(xtrain, ytrain))
print(svclassifier.score(xtest, ytest))

How to properly use sklearn's cross_validate with One Hot Encoded classes?

I created a model to classify my 8-classes dataset and get some scores from it using MLP. To do so, I decided to use sklearn.metrics.cross_validate, using 10 folds.
The following code works fine:
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import cross_validate
from sklearn.metrics import accuracy_score, make_scorer, f1_score
import pandas as pd
def MLPClasify(sample):
df = pd.read_csv('my_path\\my_file.csv', header=None)
y = df[NumberOfFeatures]
x = df.drop([NumberOfFeatures], axis=1)
clf = MLPClassifier(hidden_layer_sizes=(27), activation='logistic', max_iter=500, alpha=0.0001,
solver='adam', verbose=10, random_state=21, tol=0.000000001)
clf.out_activation_ = 'softmax'
scoring = {'Accuracy': make_scorer(accuracy_score), 'F1': make_scorer(f1_score,
average='weighted')}
scores = cross_validate(clf, x, y, cv=10, scoring=scoring)
return scores
Everything went ok, I was getting some accuracies around 60%. So I decided to use one hot encoding to see if I can get better results. So I wrote the following code:
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import cross_validate
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
from sklearn.metrics import accuracy_score, make_scorer, f1_score
import pandas as pd
def MLPClasify(sample):
df = pd.read_csv('my_path\\my_file.csv', header=None)
y = df[NumberOfFeatures]
x = df.drop([NumberOfFeatures], axis=1)
label_encoder = LabelEncoder()
integer_encoded = label_encoder.fit_transform(y)
onehot_encoder = OneHotEncoder()
integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)
onehot_encoded = onehot_encoder.fit_transform(integer_encoded)
y = onehot_encoded
clf = MLPClassifier(hidden_layer_sizes=(27), activation='logistic', max_iter=500, alpha=0.0001,
solver='adam', verbose=10, random_state=21, tol=0.000000001)
clf.out_activation_ = 'softmax'
scoring = {'Accuracy': make_scorer(accuracy_score), 'F1': make_scorer(f1_score,
average='weighted')}
scores = cross_validate(clf, x, y, cv=10, scoring=scoring)
return scores
Well, the code runs, but I get the following warning:
UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true nor predicted samples. Use zero_division parameter to con
trol this behavior.
average, "true nor predicted", 'F-score is', len(true_sum)
Also, my accuracy drops to less than 2%
Any ideas on what I may be doing wrong?
Thanks for the help

Adaboosting Ensemble learning

I am implemeting Adaboost algorithm with Decision tree from sklearn library and once i predict the results i got an error any explanation and thank you :
The error is :
AdaBoostClassifier with algorithm='SAMME.R' requires that the weak learner supports the calculation of class probabilities with a predict_proba method.
Please change the base estimator or set algorithm='SAMME' instead.
The dataset is from UCI repository you can access it from this link :
https://archive.ics.uci.edu/ml/machine-learning-databases/heart-disease/
The code is as follow:
from sklearn.model_selection import *
from sklearn.ensemble import AdaBoostClassifier
from sklearn.tree import DecisionTreeClassifier
np.random.seed(1)
selected_features=['sex', 'cp','fbs', 'exang']
X=datascaled[selected_features]
Y=datascaled['num']
# Split the dataset in two equal parts
X_train, X_test, y_train, y_test = train_test_split(
X, Y, test_size=0.3, random_state=0)
param_grid = {"base_estimator__criterion" : ["gini", "entropy"],
"base_estimator__splitter" : ["best", "random"],
"n_estimators": [1, 2]
}
DTC = DecisionTreeClassifier(random_state = 11, max_features = "auto", class_weight = "balanced",max_depth = None)
ABC = AdaBoostClassifier(base_estimator = DTC)
# run grid search
model1=GridSearchCV(ABC, param_grid=param_grid, scoring = 'accuracy')
model1.fit(X_train,y_train)
#The best hyper parameters set
print("Best Hyper Parameters:\n",model1.best_params_)
prediction=model1.predict(X_test)
#importing the metrics module
from sklearn import metrics
#evaluation(Accuracy)
print("Accuracy:",metrics.accuracy_score(prediction,y_test))
#evaluation(Confusion Metrix)
print("Confusion Metrix:\n",metrics.confusion_matrix(prediction,y_test))
#after tuning
import pandas as pd
import numpy as np
from sklearn import preprocessing
import matplotlib.pyplot as plt
plt.rc("font", size=14)
from time import *
from sklearn import metrics
n_folds=10
model=AdaBoostClassifier(random_state = 11,base_estimator= 'entropy', n_estimators= 2)
cv = 10
t0 = time()
y_pred = cross_val_predict(model, X=X, y=Y, n_jobs=-1, cv=cv)
t = time() - t0
print("=" * 52)
print("time cost: {}".format(t))
print()
print("confusion matrix\n", metrics.confusion_matrix(y, y_pred))
print()
print("\t\taccuracy: {}".format(metrics.accuracy_score(y, y_pred)))
print("\t\troc_auc_score: {}".format(metrics.roc_auc_score(y, y_pred)))
print("\t\tcohen_kappa_score: {}".format(metrics.cohen_kappa_score(y, y_pred)))
print()
print("\t\tclassification report")
print("-" * 52)
print(metrics.classification_report(y, y_pred))