I have an estimator that should be compatible with the sklearn api. I am trying to fit one parameter of this estimator with gridsearchcv but I do not understand how to do it.
This is my code:
import numpy as np
import sklearn as sk
from sklearn.linear_model import LinearRegression, LassoLarsCV, RidgeCV
from sklearn.linear_model.base import LinearClassifierMixin, SparseCoefMixin, BaseEstimator
class ELM(BaseEstimator):
def __init__(self, n_nodes, link='rbf', output_function='lasso', n_jobs=1, c=1):
self.n_jobs = n_jobs
self.n_nodes = n_nodes
self.c = c
if link == 'rbf':
self.link = lambda z: np.exp(-z*z)
elif link == 'sig':
self.link = lambda z: 1./(1 + np.exp(-z))
elif link == 'id':
self.link = lambda z: z
else:
self.link = link
if output_function == 'lasso':
self.output_function = LassoLarsCV(cv=10, n_jobs=self.n_jobs)
elif output_function == 'lr':
self.output_function = LinearRegression(n_jobs=self.n_jobs)
elif output_function == 'ridge':
self.output_function = RidgeCV(cv=10)
else:
self.output_function = output_function
return
def H(self, x):
n, p = x.shape
xw = np.dot(x, self.w.T)
xw = xw + np.ones((n, 1)).dot(self.b.T)
return self.link(xw)
def fit(self, x, y, w=None):
n, p = x.shape
self.mean_y = y.mean()
if w == None:
self.w = np.random.uniform(-self.c, self.c, (self.n_nodes, p))
else:
self.w = w
self.b = np.random.uniform(-self.c, self.c, (self.n_nodes, 1))
self.h_train = self.H(x)
self.output_function.fit(self.h_train, y)
return self
def predict(self, x):
self.h_predict = self.H(x)
return self.output_function.predict(self.h_predict)
def get_params(self, deep=True):
return {"n_nodes": self.n_nodes,
"link": self.link,
"output_function": self.output_function,
"n_jobs": self.n_jobs,
"c": self.c}
def set_params(self, **parameters):
for parameter, value in parameters.items():
setattr(self, parameter, value)
### Fit the c parameter ###
X = np.random.normal(0, 1, (100,5))
y = X[:,1] * X[:,2] + np.random.normal(0, .1, 100)
gs = sk.grid_search.GridSearchCV(ELM(n_nodes=20, output_function='lr'),
cv=5,
param_grid={"c":np.linspace(0.0001,1,10)},
fit_params={})
#gs.fit(X, y) # Error
There are 2 problems within your code:
You didn't specify scoring argument to GridSearchCV. You seems be doing regression, so mean_squared_error is an option.
Your set_params doesn't return reference to the object itself. You should add return self after the for loop.
As Andreas already mentioned, you rarely need to redefine set_params and get_params in scikit-learn. Just having inherited from the BaseEstimator should be enough.
You inherit from BaseEstimator. It should just work. See https://scikit-learn.org/dev/developers/develop.html
FYI this might be interesting to you: https://github.com/scikit-learn/scikit-learn/pull/3306
Related
I would like to make a unit test using pytest library where I need to set return_value of a patched mocked object predict method. But currently, even if I set return_value, it returns also a mock object. What would be the correct approach to this? What should be the approach to test for correct output of the fit_model method?
A toy example - in path src.model_training_service.model I have python file bol.py with contents:
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score
class bol:
def __init__(self):
self.model = None
def fit_model(self, x, y):
self.model = RandomForestClassifier()
print(type(self.model))
self.model.fit(x,y)
y_pred = self.model.predict(x) # this returns mock object instead of intended np.array([9,4,5])
print('printing y_pred')
print(type(y_pred))
acc = accuracy_score(y, y_pred)
result = 1 + acc
return result
The test is looking like this:
def test_bol(mocker):
m = mocker.patch('src.model_training_service.models.bol.RandomForestClassifier')
m.predict.return_value = np.array([9,4,5])
x = np.array([[1,2,3,4],
[5,6,7,8],
[3,2,1,1]])
y = np.array([8, 4, 2])
obj = bol()
result = obj.fit_model(x,y)
expected_accuracy = 2.
assert result == expected_accuracy
When I try to put the class and the test into one file, it works as inteded:
Contents of one file:
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score
class bol2:
def __init__(self):
self.model = None
def fit_model(self, x, y):
self.model = RandomForestClassifier()
print(type(self.model))
self.model.fit(x,y)
y_pred = self.model.predict(x)
print('printing y_pred')
print(type(y_pred))
acc = accuracy_score(y, y_pred)
result = 1 + acc
return result
def test_bol2(mocker):
m = mocker.patch('sklearn.ensemble.RandomForestClassifier') # but this is probably wrong, because I am not mocking the object in the module
m.predict.return_value = np.array([9,4,5])
x = np.array([[1,2,3,4],
[5,6,7,8],
[3,2,1,1]])
y = np.array([8, 4, 2])
obj = bol2()
expected_accuracy = 2.
result = obj.fit_model(x,y)
assert result == expected_accuracy
Thanks
I am trying to implement from scratch the multiclass logistic regression but my implementation returns bad results. I believe the definition of the gradient function and the cost function is fine. Maybe there is a problem with how these functions are interacting with the minimize function. I have tried it but I could not find out what is wrong. Could you please cast some light?
You can add the estimator 'myLR': myLR(**par_dict), with paramters
par_dict= {'alpha': 0.1, 'maxit': 2000, 'opt_method': 'bfgs', 'positive': False, 'penalty': None, 'verbose': True, 'seed': 3}
in this example or in any of these examples to test it.
import numpy as np
from scipy.optimize import minimize
from sklearn import preprocessing
class myLR():
def __init__(self, alpha=0.1, reltol=1e-8, maxit=1000, opt_method=None, verbose=True, seed=0):
self.alpha = alpha
self.maxit = maxit
self.reltol = reltol
self.seed = seed
self.verbose = verbose
self.opt_method = opt_method
self.lbin = preprocessing.LabelBinarizer()
def w_2d(self, w, n_classes):
return np.reshape(w, (-1, n_classes), order='F')
def softmax(self, W, X):
a = np.exp(X # W)
o = a / np.sum(a, axis=1, keepdims=True)
return o
def cost_wraper(self, W):
return self.cost(W, self.X, self.T, self.n_samples, self.n_classes)
def cost(self, W, X, T, n_samples, n_classes):
W = self.w_2d(W, n_classes)
log_O = np.log(self.softmax(W, X))
reg = self.apha * np.linalg.norm(W, ord='fro')
c = -np.sum([np.vdot(T[[i]], log_O[[i]]) for i in range(n_samples)]) / n_samples + reg
return c
def gradient_wraper(self, W):
return self.gradient(W, self.X, self.T, self.n_samples, self.n_classes)
def gradient(self, W, X, T, n_samples, n_classes):
W = self.w_2d(W, n_classes)
O = self.softmax(W, X)
reg = self.alpha * W
grad = -X.T.dot(T - O) / n_samples + reg
return grad.flatten()
def fit(self, X, y=None):
self.n_classes = len(np.unique(y))
self.n_samples, n_features = X.shape
if self.n_classes == 2:
self.T = np.zeros((self.n_samples, self.n_classes), dtype=np.float64)
for i, cls in enumerate(range(self.n_classes)):
self.T[y == cls, i] = 1
else:
self.T = self.lbin.fit_transform(y)
self.X = X
np.random.seed(self.seed)
W_0 = np.random.random(n_features * self.n_classes)
options = {'disp': self.verbose, 'maxiter': self.maxit}
f_min = minimize(fun=self.cost_wraper, x0=W_0,
method=self.opt_method,
jac=self.gradient_wraper,
options=options)
self.coef_ = self.w_2d(f_min.x, self.n_classes)
self.W_ = self.coef_
return self
def predict_proba(self, X):
O = self.softmax(self.coef_, X)
return O
def predict(self, X):
sigma = self.predict_proba(X)
y_pred = np.argmax(sigma, axis=1)
return y_pred
Edit: Regularization term is included.
I think it is now working with the following code.
import numpy as np
from scipy.optimize import minimize
from sklearn import preprocessing
class myLR():
def __init__(self, reltol=1e-8, maxit=1000, opt_method=None, verbose=True, seed=0):
self.maxit = maxit
self.reltol = reltol
self.seed = seed
self.verbose = verbose
self.opt_method = opt_method
self.lbin = preprocessing.LabelBinarizer()
def w_2d(self, w, n_classes):
return np.reshape(w, (n_classes, -1))
def softmax(self, W, X):
a = np.exp(X # W.T)
o = a / np.sum(a, axis=1, keepdims=True)
return o
def squared_norm(self, x):
x = np.ravel(x, order='K')
return np.dot(x, x)
def cost(self, W, X, T, n_samples, n_classes):
W = self.w_2d(W, n_classes)
log_O = np.log(self.softmax(W, X))
c = -(T * log_O).sum()
return c / n_samples
def gradient(self, W, X, T, n_samples, n_classes):
W = self.w_2d(W, n_classes)
O = self.softmax(W, X)
grad = -(T - O).T.dot(X)
return grad.ravel() / n_samples
def fit(self, X, y=None):
n_classes = len(np.unique(y))
n_samples, n_features = X.shape
if n_classes == 2:
T = np.zeros((n_samples, n_classes), dtype=np.float64)
for i, cls in enumerate(np.unique(y)):
T[y == cls, i] = 1
else:
T = self.lbin.fit_transform(y)
np.random.seed(self.seed)
W_0 = np.random.random((self.n_classes, self.n_features))
options = {'disp': self.verbose, 'maxiter': self.maxit}
f_min = minimize(fun=self.cost, x0=W_0,
args=(X, T, n_samples, n_classes),
method=self.opt_method,
jac=self.gradient,
options=options)
self.coef_ = self.w_2d(f_min.x, n_classes)
self.W_ = self.coef_
return self
def predict_proba(self, X):
O = self.softmax(self.W_, X)
return O
def predict(self, X):
sigma = self.predict_proba(X)
y_pred = np.argmax(sigma, axis=1)
return y_pred
I am trying to create a custom estimator based on scikit learn. I have written the below dummy code to explain my problem. In the score method, I am trying to access mean_ calulated in fit. But I am unable to. What I am doing wrong? I have tried many things and have done this referring three four articles. But didn't find the issue.
I have read the documentation and did few changes. But nothing worked. I have also tried inheriting BaseEstimator, ClassifierMixin. But that also didn't work.
This a dummy program. Don't go by what it is trying to do.
import numpy as np
from sklearn.model_selection import cross_val_score
class FilterElems:
def __init__(self, thres):
self.thres = thres
def fit(self, X, y=None, **kwargs):
self.mean_ = np.mean(X)
self.std_ = np.std(X)
return self
def predict(self, X):
# return sign(self.predict(inputs))
X = (X - self.mean_) / self.std_
return X[X > self.thres]
def get_params(self, deep=False):
return {'thres': self.thres}
def score(self, *x):
print(self.mean_) # errors out, mean_ and std_ are wiped out
if len(x[1]) > 50:
return 1.0
else:
return 0.5
model = FilterElems(thres=0.5)
print(cross_val_score(model,
np.random.randint(1, 1000, (100, 100)),
None,
scoring=model.score,
cv=5))
Err:
AttributeError: 'FilterElems' object has no attribute 'mean_'
You are almost there.
The signature for scorer is scorer(estimator, X, y). The cross_val_score calls the scorer method by passing the estimator object as the first parameter. Since your signature of scorer is a variable argument function, the first item will hold the estimator
change your score to
def score(self, *x):
print(x[0].mean_)
if len(x[1]) > 50:
return 1.0
else:
return 0.5
Working code
import numpy as np
from sklearn.model_selection import cross_val_score
class FilterElems:
def __init__(self, thres):
self.thres = thres
def fit(self, X, y=None, **kwargs):
self.mean_ = np.mean(X)
self.std_ = np.std(X)
return self
def predict(self, X):
X = (X - self.mean_) / self.std_
return X[X > self.thres]
def get_params(self, deep=False):
return {'thres': self.thres}
def score(self, estimator, *x):
print(estimator.mean_, estimator.std_)
if len(x[0]) > 50:
return 1.0
else:
return 0.5
model = FilterElems(thres=0.5)
print(cross_val_score(model,
np.random.randint(1, 1000, (100, 100)),
None,
scoring=model.score,
cv=5))
Outout
504.750125 288.84916035447355
501.7295 289.47825925231416
503.743375 288.8964170227962
503.0325 287.8292687406025
500.041 289.3488678377712
[0.5 0.5 0.5 0.5 0.5]
The input for scoring param in cross_val_score needs to str or callable with signature scoring(estimator, X, y). In your case, you don't seems to need the y, hence you can leave that in your callable. Also, you need to ensure that the output of the score has to be single value.
The solution would look something like this for your problem.
import numpy as np
from sklearn.model_selection import cross_val_score
from sklearn.base import TransformerMixin
class FilterElems(TransformerMixin):
def __init__(self, thres):
self.thres = thres
def fit(self, X, y=None, **kwargs):
self.mean_ = np.mean(X)
self.std_ = np.std(X)
return self
def predict(self, X):
# return sign(self.predict(inputs))
X = (X - self.mean_) / self.std_
return X[X > self.thres]
def get_params(self, deep=False):
return {'thres': self.thres}
def scorer(tranformer, X):
print(tranformer.mean_) # Now it prints out, mean_ and std_
result=[]
for x in X:
# do the stuff you want here
if x[1] > 50:
result.append(1)
else:
result.append(0.5)
# but return a single value
return np.mean(result)
np.random.seed(1)
model = FilterElems(thres=0.5)
print(cross_val_score(model,
np.random.randint(1, 1000, (100, 100)),
None,
scoring=scorer,
cv=5))
# [0.95 1. 1. 0.975 0.975]
I am making a prediction and implementing a neural network, is currently working with the numpy library and I am adapting the code to the data that I have.
I leave the current progress of the neural network, I have an error at the end of the code and I do not understand it well.
Anyone who can help me please?
import numpy as np
from sklearn.cross_validation import train_test_split
class LinearLayer:
def __init__(self, n_input, n_output):
self.n = n_input
self.m = n_output
self.W = (1/np.sqrt(n_input))*np.random.rand(n_input+1, n_output)
def forward(self, X):
self.input = np.zeros((X.shape[0],self.n+1))
# if only one feature, the input should always be a batch, at least
if len(X.shape) == 1: # of one element
self.input[:-1,:] = X.reshape(-1,self.n)
else:
self.input[:,:-1] = X
self.input[:,-1] = 1
self.output = self.input.dot(self.W) # xW + b
return self.output
def backward(self, d_out):
self.gradients = self.W.dot(d_out)[:-1]
self.dW = np.einsum("ij,ki", self.input, d_out)
return self.gradients
def updateWeights(self, lr=0.1):
self.W = self.W - lr*self.dW
class Sigmoid:
def __init__(self, n_input):
self.output = np.zeros(n_input)
self.gradients = np.zeros(n_input)
def forward(self, X):
self.output = 1/(np.exp(-X)+1)
return self.output
def backward(self, d_out):
ds = self.output.T*(1 - self.output).T
self.gradients = ds*d_out
return self.gradients
print("Training a multilayer perceptron\n")
import pandas as pd
data = pd.read_csv('Data_Balanceada.csv') #Data (74,11)
X = data.iloc[:,0:11]
y = data.iloc[:,-1]
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.25, random_state=1)
h1 = LinearLayer(11,1) #stack some layers
s1 = Sigmoid(7)
h2 = LinearLayer(7,1)
s2 = Sigmoid(1)
def loss(pred, target):
return np.mean(np.power(pred-target,2))
predict = lambda x: s2.forward(h2.forward(s1.forward(h1.forward(x))))
backpropagate = lambda d: h1.backward(s1.backward(h2.backward(s2.backward(d))))
lr = 0.005
n = 0 # patience
max_epochs = 1500
valid = loss(predict(X_test), y_test)
for i in range(max_epochs):
l = 0
p = predict(X_train)
backpropagate(p.T-y_train.T)
h1.updateWeights(lr)
h2.updateWeights(lr)
l = loss(p,y_train)
new_valid = loss(predict(X_test), y_test)
if new_valid < valid:
valid = new_valid
n = 0
else:
n += 1
if n > 50: break
if i%50 == 0:
print("Loss: {0}\t\tValidation: {1}".format(l/100, valid))
lr = lr*0.97
# Validation
print("\nFinal validation loss: {0}. {1} epochs\n".format(loss(predict(X_test), y_test),i+1))
#print(np.argmax(predict(X_test), axis=1))
#print(np.argmax(y_test, axis=1))
link Dataset:
https://mega.nz/#!jM8AQAbB!61NOeJadGXtiKJQsn_tdJ955p5lRD6kQjBlCQTHtt6I
I have this error:
Data must be 1-dimensional
IMG - ERROR
I try to feed the data sample-by-sample. The result is either completely wrong or very approximate (25-50% absolute error) on different datasets. The result is fine for all datasets, if training in one go.
import itertools as itools
import numpy as np
import tensorflow as tf
from sklearn import preprocessing
class Test:
def __init__(self, x, y):
self.x = x
self.y = y
self._i = 0
def do_test(self):
x_col = tf.contrib.layers.real_valued_column("x", dimension=1)
model = tf.contrib.learn.LinearRegressor(feature_columns=[x_col])
print("Fitting")
max_steps = 80
for _ in range(0, len(self.x)):
model.fit(input_fn=self.input_split, steps=max_steps)
print("Predicting")
scaled_out = model.predict(input_fn=self.eval_fn)
print(self._inverse_y(list(itools.islice(scaled_out, self.eval_len))))
def input_split(self):
if 0 == self._i:
self.x_std, self.y_std = self._transform(self.x, self.y)
if len(self.x_std) == self._i:
raise StopIteration
x = self.x_std[self._i]
y = self.y_std[self._i]
self._i += 1
feature_cols = {"x": tf.constant([x], dtype=tf.float32),
}
print(x, y)
label = tf.constant([y], dtype=tf.float32)
return feature_cols, label
def eval_fn(self):
x = [0, 1, 5, 10]
y = np.zeros(len(x))
self.eval_len = len(x)
x_std, y_std = self._transform(x, y)
feature_cols = {"x": tf.constant(x_std, dtype=tf.float32),
}
label = tf.constant(y_std, dtype=tf.float32)
return feature_cols, label
def _transform(self, x_in, y_in):
if not hasattr(self, "x_scaler"):
self.x_scaler = preprocessing.StandardScaler().fit(x_in)
self.y_scaler = preprocessing.StandardScaler().fit(y_in)
x_std = self.x_scaler.transform(x_in)
y_std = self.y_scaler.transform(y_in)
return x_std, y_std
def _inverse_y(self, y_std):
return self.y_scaler.inverse_transform(y_std)
P.S. fit and partial_fit are the same according to the source
This looks like learning_rate and/or optimization. Please try with them as follows:
model = tf.contrib.learn.LinearRegressor(..., optimizer=tf.train.YOUR_OPTIMIZER(YOUR_LEARNING_RATE)))