We Keep Coding

How to incorporate recursive VIF elimination into scikit-learn's pipeline?

I am trying to implement a custom pipeline into the pipeline object of scikit-learn. The pipeline is to recursively eliminate features using VIF. I referenced to the codes here
class ReduceVIF(base.BaseEstimator, base.TransformerMixin):
def __init__(self, thresh=10.0):
# From looking at documentation, values between 5 and 10 are "okay".
# Above 10 is too high and so should be removed.
self.thresh = thresh
self.scaler = preprocessing.StandardScaler()
def fit(self, X, y=None):
X_copy = X.copy()
print("ReduceVIF fit")
if hasattr(self, 'scaler'):
X = self.scaler.fit(X)
X = ReduceVIF.calculate_vif(X, self.thresh)
print(X)
self.predictors = X.columns
return self
def transform(self, X, y=None):
print("ReduceVIF transform")
columns = self.predictors
if hasattr(self, 'scaler'):
X = pd.DataFrame(self.scaler.transform(X), columns=columns)
return X
# return ReduceVIF.calculate_vif(X, self.thresh)
#staticmethod
def calculate_vif(X, thresh=10.0):
# Taken from https://stats.stackexchange.com/a/253620/53565 and modified
dropped = True
count = 0
while dropped and count <= 15:
print(count)
variables = X.columns
dropped = False
vif = [
variance_inflation_factor(X[variables].values, X.columns.get_loc(var))
for var in X.columns
]
max_vif = max(vif)
if max_vif > thresh:
maxloc = vif.index(max_vif)
print(f"Dropping {X.columns[maxloc]} with vif={max_vif}")
X = X.drop([X.columns.tolist()[maxloc]], axis=1)
dropped = True
count +=1
print(X.shape)
return X
and I tried to call/make a pipeline as such
# create a feature preparation pipeline for a model
def make_finetuning_pipeline(model):
steps = list()
# standardization
#steps.append(('standardize', preprocessing.StandardScaler()))
steps.append(('remove_multicollinearity', ReduceVIF(thresh=10)))
#steps.append(("feature_selection", feature_selection.RFE(linear_model.LogisticRegression(penalty='l1', solver='liblinear'))))
# the model
steps.append(('model', model))
# create pipeline
_pipeline = pipeline.Pipeline(steps=steps)
return _pipeline
But somehow it is not working and the error is either each fold has different columns, or there is an attribute error. Does anyone know how to incoprorate VIF into a scikit learn pipeline?
Here is my snippet of code in github gist for reproducibility. Github Gist

Answer to my own question: I have spent half a day trying to debug, a preliminary version that works is as follows, not elegant, but for now it is working as expected.
class ReduceVIF(base.BaseEstimator, base.TransformerMixin):
def __init__(self, thresh=10):
# From looking at documentation, values between 5 and 10 are "okay".
# Above 10 is too high and so should be removed.
self.thresh = thresh
self.predictor_cols = [
"radius_mean",
"texture_mean",
"perimeter_mean",
"area_mean",
"smoothness_mean",
"compactness_mean",
"concavity_mean",
"concave points_mean",
"symmetry_mean",
"fractal_dimension_mean",
"radius_se",
"texture_se",
"perimeter_se",
"area_se",
"smoothness_se",
"compactness_se",
"concavity_se",
"concave points_se",
"symmetry_se",
"fractal_dimension_se",
"radius_worst",
"texture_worst",
"perimeter_worst",
"area_worst",
"smoothness_worst",
"compactness_worst",
"concavity_worst",
"concave points_worst",
"symmetry_worst",
"fractal_dimension_worst",
]
def reset(self):
self.predictor_cols = [
"radius_mean",
"texture_mean",
"perimeter_mean",
"area_mean",
"smoothness_mean",
"compactness_mean",
"concavity_mean",
"concave points_mean",
"symmetry_mean",
"fractal_dimension_mean",
"radius_se",
"texture_se",
"perimeter_se",
"area_se",
"smoothness_se",
"compactness_se",
"concavity_se",
"concave points_se",
"symmetry_se",
"fractal_dimension_se",
"radius_worst",
"texture_worst",
"perimeter_worst",
"area_worst",
"smoothness_worst",
"compactness_worst",
"concavity_worst",
"concave points_worst",
"symmetry_worst",
"fractal_dimension_worst",
]
def fit(self, X, y=None):
print("ReduceVIF fit")
tmp, self.predictor_cols = ReduceVIF.calculate_vif(X, self.predictor_cols, self.thresh)
col_index = [self.predictor_cols.index(col_name) for col_name in self.predictor_cols]
self.col_index = col_index
print("tmp", self.col_index)
self.reset()
return self
def transform(self, X, y=None):
print("ReduceVIF transform")
# columns = X.columns.tolist()
# print(X.shape)
return X[:, self.col_index]
#staticmethod
def calculate_vif(X, columns, thresh=10.0):
# Taken from https://stats.stackexchange.com/a/253620/53565 and modified
dropped = True
count = 0
while dropped and count <= 15:
column_index = X.shape[1]
predictor_cols = np.arange(X.shape[1])
dropped = False
print(count)
vif = []
for var in range(column_index):
# print(predictor_cols.shape)
vif.append(variance_inflation_factor(X[:, predictor_cols], var))
max_vif = max(vif)
if max_vif > thresh:
maxloc = vif.index(max_vif)
print(f"Dropping {maxloc} with vif={max_vif}")
# X = X.drop([X.columns.tolist()[maxloc]], axis=1)
X = np.delete(X, maxloc, axis=1)
columns.pop(maxloc)
dropped = True
count += 1
return X, columns

ModuleNotFoundError: No module named 'grad'

I try to run this Neural Network script (for a regression model)
There are two classes defined above. One is Standardizer class and other is Neural Net class. The Standardizer class normalizes all the values and the NeuralNet class builds the neural network that learns the data through feed forward and back propagation.
This function takes the the number of inputs, hidden units, and outputs as the three parameters.
The set_hunit function is used to either update or initiate the weights.It takes the weight as the parameter.
The Pack function packs the multiple weights of each layer into one vector. The unpack function does vice versa.
Forward pass in neural network propagates as shown below:
𝑍𝑌=ℎ(𝑋𝑙⋅𝑉)=𝑍𝑙⋅𝑊
Activation function is used to make the network non linear. We may use tanh or RBG or etc.
In the backward pass the function takes the the z values, Target values and the error as input. Based on the delta value, the weights and the bias are updated accoringly. This method returns the weight vector packed together of that particualr layer. Below are the functions that are excecuted during backward pass.
𝑉𝑊←𝑉+𝛼ℎ1𝑁1𝐾𝑋𝑙⊤((𝑇−𝑌)𝑊⊤⊙(1−𝑍2))←𝑊+𝛼𝑜1𝑁1𝐾𝑍𝑙⊤(𝑇−𝑌)
The train function takes the feautures and the target as the input. The gradientf unpacks the weights,proceeds with the forward pass by calling forward function. Now error is calculated using results of forward pass. Now back propagation is proceeded by calling backward function with parameters as error, Z, T(Target), _lambda.
The optimtarget function tries to reduce the error by using the object function and updates the weights accordingly.
The use method is applied to the test data after training the model. Testing data is passed as parameter and it stadardizes the data. Then forward is applied on the data which returns the predictions
This shows module not found error, but I have installed grad module with pip installation
#Importing required libraries
import pandas as pd
import numpy as np
import seaborn as sns
import grad
import matplotlib.pyplot as plt
# Reading data using pandas library
vehicle_data=pd.read_csv('processed_Data.csv')
# Overall idea about distribution of data
vehicle_data.hist(bins=40, figsize=(20,15))
plt.show()
# Count plot of Ellectric Range
sns.countplot(x='Electric Range',data=vehicle_data)
# Joint plot between Latitude on x axis and Longitude on y axis
sns.jointplot(x=vehicle_data.BaseMSRP.values,y=vehicle_data.LegislativeDistrict.values,height=10)
plt.xlabel("Base MSRP",fontsize=10)
plt.ylabel("Lengislative District",fontsize=10)
# function to drop the rows that has null or missing values
vehicle_data=vehicle_data.dropna()
# Data is already clean and has no missing values
vehicle_data.shape
#Dropping unwanted columns
vehicle_data=vehicle_data.drop(['VIN (1-10)','County', 'City', 'State', 'ZIP Code', 'DOL Vehicle ID'],axis=1)
vehicle_data.shape
# Seperating target variable
t=pd.DataFrame(vehicle_data.iloc[:,8])
vehicle_data=vehicle_data.drop(['Electric Range'],axis=1)
t
vehicle_data.head()
#NeuralNet class for regression
# standardization class
class Standardizer:
""" class version of standardization """
def __init__(self, X, explore=False):
self._mu = np.mean(X,8)
self._sigma = np.std(X,8)
if explore:
print ("mean: ", self._mu)
print ("sigma: ", self._sigma)
print ("min: ", np.min(X,8))
print ("max: ", np.max(X,8))
def set_sigma(self, s):
self._sigma[:] = s
def standardize(self,X):
return (X - self._mu) / self._sigma
def unstandardize(self,X):
return (X * self._sigma) + self._mu
def add_ones(w):
return np.hstack((np.ones((w.shape[8], 1)), w))
from grad import scg, steepest
from copy import copy
class NeuralNet:
def __init__(self, nunits):
self._nLayers=len(nunits)-1
self.rho = [1] * self._nLayers
self._W = []
wdims = []
lenweights = 0
for i in range(self._nLayers):
nwr = nunits[i] + 1
nwc = nunits[i+1]
wdims.append((nwr, nwc))
lenweights = lenweights + nwr * nwc
self._weights = np.random.uniform(-0.1,0.1, lenweights)
start = 0 # fixed index error 20110107
for i in range(self._nLayers):
end = start + wdims[i][0] * wdims[i][1]
self._W.append(self._weights[start:end])
self._W[i].resize(wdims[i])
start = end
self.stdX = None
self.stdT = None
self.stdTarget = True
def add_ones(self, w):
return np.hstack((np.ones((w.shape[8], 1)), w))
def get_nlayers(self):
return self._nLayers
def set_hunit(self, w):
for i in range(self._nLayers-1):
if w[i].shape != self._W[i].shape:
print("set_hunit: shapes do not match!")
break
else:
self._W[i][:] = w[i][:]
def pack(self, w):
return np.hstack(map(np.ravel, w))
def unpack(self, weights):
self._weights[:] = weights[:] # unpack
def cp_weight(self):
return copy(self._weights)
def RBF(self, X, m=None,s=None):
if m is None: m = np.mean(X)
if s is None: s = 2 #np.std(X)
r = 1. / (np.sqrt(2*np.pi)* s)
return r * np.exp(-(X - m) ** 2 / (2 * s ** 2))
def forward(self,X):
t = X
Z = []
for i in range(self._nLayers):
Z.append(t)
if i == self._nLayers - 1:
t = np.dot(self.add_ones(t), self._W[i])
else:
t = np.tanh(np.dot(self.add_ones(t), self._W[i]))
#t = self.RBF(np.dot(np.hstack((np.ones((t.shape[0],1)),t)),self._W[i]))
return (t, Z)
def backward(self, error, Z, T, lmb=0):
delta = error
N = T.size
dws = []
for i in range(self._nLayers - 1, -1, -1):
rh = float(self.rho[i]) / N
if i==0:
lmbterm = 0
else:
lmbterm = lmb * np.vstack((np.zeros((1, self._W[i].shape[1])),
self._W[i][1:,]))
dws.insert(0,(-rh * np.dot(self.add_ones(Z[i]).T, delta) + lmbterm))
if i != 0:
delta = np.dot(delta, self._W[i][1:, :].T) * (1 - Z[i]**2)
return self.pack(dws)
def _errorf(self, T, Y):
return T - Y
def _objectf(self, T, Y, wpenalty):
return 0.5 * np.mean(np.square(T - Y)) + wpenalty
def train(self, X, T, **params):
verbose = params.pop('verbose', False)
# training parameters
_lambda = params.pop('Lambda', 0.)
#parameters for scg
niter = params.pop('niter', 1000)
wprecision = params.pop('wprecision', 1e-10)
fprecision = params.pop('fprecision', 1e-10)
wtracep = params.pop('wtracep', False)
ftracep = params.pop('ftracep', False)
# optimization
optim = params.pop('optim', 'scg')
if self.stdX == None:
explore = params.pop('explore', False)
self.stdX = Standardizer(X, explore)
Xs = self.stdX.standardize(X)
if self.stdT == None and self.stdTarget:
self.stdT = Standardizer(T)
T = self.stdT.standardize(T)
def gradientf(weights):
self.unpack(weights)
Y,Z = self.forward(Xs)
error = self._errorf(T, Y)
return self.backward(error, Z, T, _lambda)
def optimtargetf(weights):
""" optimization target function : MSE
"""
self.unpack(weights)
#self._weights[:] = weights[:] # unpack
Y,_ = self.forward(Xs)
Wnb=np.array([])
for i in range(self._nLayers):
if len(Wnb)==0: Wnb=self._W[i][1:,].reshape(self._W[i].size-self._W[i][0,].size,1)
else: Wnb = np.vstack((Wnb,self._W[i][1:,].reshape(self._W[i].size-self._W[i][0,].size,1)))
wpenalty = _lambda * np.dot(Wnb.flat ,Wnb.flat)
return self._objectf(T, Y, wpenalty)
if optim == 'scg':
result = scg(self.cp_weight(), gradientf, optimtargetf,
wPrecision=wprecision, fPrecision=fprecision,
nIterations=niter,
wtracep=wtracep, ftracep=ftracep,
verbose=False)
self.unpack(result['w'][:])
self.f = result['f']
elif optim == 'steepest':
result = steepest(self.cp_weight(), gradientf, optimtargetf,
nIterations=niter,
xPrecision=wprecision, fPrecision=fprecision,
xtracep=wtracep, ftracep=ftracep )
self.unpack(result['w'][:])
if ftracep:
self.ftrace = result['ftrace']
if 'reason' in result.keys() and verbose:
print(result['reason'])
return result
def use(self, X, retZ=False):
if self.stdX:
Xs = self.stdX.standardize(X)
else:
Xs = X
Y, Z = self.forward(Xs)
if self.stdT is not None:
Y = self.stdT.unstandardize(Y)
if retZ:
return Y, Z
return Y

Try to open command prompt and type pip install grad or if you using jupyter notebook, make a new code shell and type !pip install grad before you importing it
Hope that solves your problem

Custom scikit-learn scorer can't access mean after fit

I am trying to create a custom estimator based on scikit learn. I have written the below dummy code to explain my problem. In the score method, I am trying to access mean_ calulated in fit. But I am unable to. What I am doing wrong? I have tried many things and have done this referring three four articles. But didn't find the issue.
I have read the documentation and did few changes. But nothing worked. I have also tried inheriting BaseEstimator, ClassifierMixin. But that also didn't work.
This a dummy program. Don't go by what it is trying to do.
import numpy as np
from sklearn.model_selection import cross_val_score
class FilterElems:
def __init__(self, thres):
self.thres = thres
def fit(self, X, y=None, **kwargs):
self.mean_ = np.mean(X)
self.std_ = np.std(X)
return self
def predict(self, X):
# return sign(self.predict(inputs))
X = (X - self.mean_) / self.std_
return X[X > self.thres]
def get_params(self, deep=False):
return {'thres': self.thres}
def score(self, *x):
print(self.mean_) # errors out, mean_ and std_ are wiped out
if len(x[1]) > 50:
return 1.0
else:
return 0.5
model = FilterElems(thres=0.5)
print(cross_val_score(model,
np.random.randint(1, 1000, (100, 100)),
None,
scoring=model.score,
cv=5))
Err:
AttributeError: 'FilterElems' object has no attribute 'mean_'

You are almost there.
The signature for scorer is scorer(estimator, X, y). The cross_val_score calls the scorer method by passing the estimator object as the first parameter. Since your signature of scorer is a variable argument function, the first item will hold the estimator
change your score to
def score(self, *x):
print(x[0].mean_)
if len(x[1]) > 50:
return 1.0
else:
return 0.5
Working code
import numpy as np
from sklearn.model_selection import cross_val_score
class FilterElems:
def __init__(self, thres):
self.thres = thres
def fit(self, X, y=None, **kwargs):
self.mean_ = np.mean(X)
self.std_ = np.std(X)
return self
def predict(self, X):
X = (X - self.mean_) / self.std_
return X[X > self.thres]
def get_params(self, deep=False):
return {'thres': self.thres}
def score(self, estimator, *x):
print(estimator.mean_, estimator.std_)
if len(x[0]) > 50:
return 1.0
else:
return 0.5
model = FilterElems(thres=0.5)
print(cross_val_score(model,
np.random.randint(1, 1000, (100, 100)),
None,
scoring=model.score,
cv=5))
Outout
504.750125 288.84916035447355
501.7295 289.47825925231416
503.743375 288.8964170227962
503.0325 287.8292687406025
500.041 289.3488678377712
[0.5 0.5 0.5 0.5 0.5]

The input for scoring param in cross_val_score needs to str or callable with signature scoring(estimator, X, y). In your case, you don't seems to need the y, hence you can leave that in your callable. Also, you need to ensure that the output of the score has to be single value.
The solution would look something like this for your problem.
import numpy as np
from sklearn.model_selection import cross_val_score
from sklearn.base import TransformerMixin
class FilterElems(TransformerMixin):
def __init__(self, thres):
self.thres = thres
def fit(self, X, y=None, **kwargs):
self.mean_ = np.mean(X)
self.std_ = np.std(X)
return self
def predict(self, X):
# return sign(self.predict(inputs))
X = (X - self.mean_) / self.std_
return X[X > self.thres]
def get_params(self, deep=False):
return {'thres': self.thres}
def scorer(tranformer, X):
print(tranformer.mean_) # Now it prints out, mean_ and std_
result=[]
for x in X:
# do the stuff you want here
if x[1] > 50:
result.append(1)
else:
result.append(0.5)
# but return a single value
return np.mean(result)
np.random.seed(1)
model = FilterElems(thres=0.5)
print(cross_val_score(model,
np.random.randint(1, 1000, (100, 100)),
None,
scoring=scorer,
cv=5))
# [0.95 1. 1. 0.975 0.975]

Does tensorflow support online training?

I try to feed the data sample-by-sample. The result is either completely wrong or very approximate (25-50% absolute error) on different datasets. The result is fine for all datasets, if training in one go.
import itertools as itools
import numpy as np
import tensorflow as tf
from sklearn import preprocessing
class Test:
def __init__(self, x, y):
self.x = x
self.y = y
self._i = 0
def do_test(self):
x_col = tf.contrib.layers.real_valued_column("x", dimension=1)
model = tf.contrib.learn.LinearRegressor(feature_columns=[x_col])
print("Fitting")
max_steps = 80
for _ in range(0, len(self.x)):
model.fit(input_fn=self.input_split, steps=max_steps)
print("Predicting")
scaled_out = model.predict(input_fn=self.eval_fn)
print(self._inverse_y(list(itools.islice(scaled_out, self.eval_len))))
def input_split(self):
if 0 == self._i:
self.x_std, self.y_std = self._transform(self.x, self.y)
if len(self.x_std) == self._i:
raise StopIteration
x = self.x_std[self._i]
y = self.y_std[self._i]
self._i += 1
feature_cols = {"x": tf.constant([x], dtype=tf.float32),
}
print(x, y)
label = tf.constant([y], dtype=tf.float32)
return feature_cols, label
def eval_fn(self):
x = [0, 1, 5, 10]
y = np.zeros(len(x))
self.eval_len = len(x)
x_std, y_std = self._transform(x, y)
feature_cols = {"x": tf.constant(x_std, dtype=tf.float32),
}
label = tf.constant(y_std, dtype=tf.float32)
return feature_cols, label
def _transform(self, x_in, y_in):
if not hasattr(self, "x_scaler"):
self.x_scaler = preprocessing.StandardScaler().fit(x_in)
self.y_scaler = preprocessing.StandardScaler().fit(y_in)
x_std = self.x_scaler.transform(x_in)
y_std = self.y_scaler.transform(y_in)
return x_std, y_std
def _inverse_y(self, y_std):
return self.y_scaler.inverse_transform(y_std)
P.S. fit and partial_fit are the same according to the source

This looks like learning_rate and/or optimization. Please try with them as follows:
model = tf.contrib.learn.LinearRegressor(..., optimizer=tf.train.YOUR_OPTIMIZER(YOUR_LEARNING_RATE)))

how to use GridSearchCV with custom estimator in sklearn?

I have an estimator that should be compatible with the sklearn api. I am trying to fit one parameter of this estimator with gridsearchcv but I do not understand how to do it.
This is my code:
import numpy as np
import sklearn as sk
from sklearn.linear_model import LinearRegression, LassoLarsCV, RidgeCV
from sklearn.linear_model.base import LinearClassifierMixin, SparseCoefMixin, BaseEstimator
class ELM(BaseEstimator):
def __init__(self, n_nodes, link='rbf', output_function='lasso', n_jobs=1, c=1):
self.n_jobs = n_jobs
self.n_nodes = n_nodes
self.c = c
if link == 'rbf':
self.link = lambda z: np.exp(-z*z)
elif link == 'sig':
self.link = lambda z: 1./(1 + np.exp(-z))
elif link == 'id':
self.link = lambda z: z
else:
self.link = link
if output_function == 'lasso':
self.output_function = LassoLarsCV(cv=10, n_jobs=self.n_jobs)
elif output_function == 'lr':
self.output_function = LinearRegression(n_jobs=self.n_jobs)
elif output_function == 'ridge':
self.output_function = RidgeCV(cv=10)
else:
self.output_function = output_function
return
def H(self, x):
n, p = x.shape
xw = np.dot(x, self.w.T)
xw = xw + np.ones((n, 1)).dot(self.b.T)
return self.link(xw)
def fit(self, x, y, w=None):
n, p = x.shape
self.mean_y = y.mean()
if w == None:
self.w = np.random.uniform(-self.c, self.c, (self.n_nodes, p))
else:
self.w = w
self.b = np.random.uniform(-self.c, self.c, (self.n_nodes, 1))
self.h_train = self.H(x)
self.output_function.fit(self.h_train, y)
return self
def predict(self, x):
self.h_predict = self.H(x)
return self.output_function.predict(self.h_predict)
def get_params(self, deep=True):
return {"n_nodes": self.n_nodes,
"link": self.link,
"output_function": self.output_function,
"n_jobs": self.n_jobs,
"c": self.c}
def set_params(self, **parameters):
for parameter, value in parameters.items():
setattr(self, parameter, value)
### Fit the c parameter ###
X = np.random.normal(0, 1, (100,5))
y = X[:,1] * X[:,2] + np.random.normal(0, .1, 100)
gs = sk.grid_search.GridSearchCV(ELM(n_nodes=20, output_function='lr'),
cv=5,
param_grid={"c":np.linspace(0.0001,1,10)},
fit_params={})
#gs.fit(X, y) # Error

There are 2 problems within your code:
You didn't specify scoring argument to GridSearchCV. You seems be doing regression, so mean_squared_error is an option.
Your set_params doesn't return reference to the object itself. You should add return self after the for loop.
As Andreas already mentioned, you rarely need to redefine set_params and get_params in scikit-learn. Just having inherited from the BaseEstimator should be enough.

You inherit from BaseEstimator. It should just work. See https://scikit-learn.org/dev/developers/develop.html
FYI this might be interesting to you: https://github.com/scikit-learn/scikit-learn/pull/3306