I have a dataframe as shown below:
vector_a vector_b
[1,2,3] [2,5,6]
[0,2,1] [2,9,1]
[4,7,1] [1,7,4]
I would like to do sklearn's cosine_similarity between the columns vector_a and vector_b to get a new column called 'cosine_distance' in the same dataframe. Do note that vector_a and vector_b are pandas df columns of list.
This is what I have attempted:
df['vector_a'] = df['vector_a'].apply(lambda x: np.asarray(x))
df['vector_b'] = df['vector_b'].apply(lambda x: np.asarray(x))
df['cosine_distance'] = cosine_similarity(df['vector_a'].apply(lambda x: np.transpose(x)),
df['vector_b'].apply(lambda x: np.transpose(x)))
And I got this error:
---> 58 df['cosine_distance'] = cosine_similarity(df['vector_a'].apply(lambda x: np.transpose(x)), df['vector_b'].apply(lambda x: np.transpose(x)))
~\Anaconda3\lib\site-packages\sklearn\metrics\pairwise.py in cosine_similarity(X, Y, dense_output)
1025 # to avoid recursive import
1026
-> 1027 X, Y = check_pairwise_arrays(X, Y)
1028
1029 X_normalized = normalize(X, copy=True)
~\Anaconda3\lib\site-packages\sklearn\metrics\pairwise.py in check_pairwise_arrays(X, Y, precomputed, dtype)
110 else:
111 X = check_array(X, accept_sparse='csr', dtype=dtype,
--> 112 estimator=estimator)
113 Y = check_array(Y, accept_sparse='csr', dtype=dtype,
114 estimator=estimator)
~\Anaconda3\lib\site-packages\sklearn\utils\validation.py in check_array(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, warn_on_dtype, estimator)
494 try:
495 warnings.simplefilter('error', ComplexWarning)
--> 496 array = np.asarray(array, dtype=dtype, order=order)
497 except ComplexWarning:
498 raise ValueError("Complex data not supported\n"
~\Anaconda3\lib\site-packages\numpy\core\numeric.py in asarray(a, dtype, order)
536
537 """
--> 538 return array(a, dtype, copy=False, order=order)
539
540
ValueError: setting an array element with a sequence.
Thank you in advance!
TLDR:
df['cosine_similarity'] = df.apply(
lambda row: cosine_similarity([row['vector_a']], [row['vector_b']])[0][0],
axis=1)
Explanation:
cosine_similarity expects 2D np.array, or list of lists. It doesn't know how to interpret pd.Series of lists. However, even if we did convert it to list of lists, the next problem arises:
cosine_similarity returns all-vs-all similarity. So, let's limit to pairwise comparison, artificially creating second dimension (note the extra square brackets in [row['vector_a']], [row['vector_b']]), and then taking the only element of a 1x1 array (zeros at the end of cosine_similarity(...)[0][0])
Related
I pass the predictors from an imputation pipeline,
I check the columns for NaN and inf values with
col_name = X.columns.to_series()[np.isinf(X).any()]
There are no columns with missing values or inf values.
classifier = MLPClassifier
I do: clf.fit(X,y) then get the error ValueError: Input contains NaN, infinity or a value too large for dtype('float64')
What am I missing here? Can anybody help or guide me as to what to do????
What might be the other possible explanations for this error? What else should I be checking for? Also
full traceback:
Input In [137], in <cell line: 4>()
1 #l = list(X.isin([np.inf, -np.inf]))
2 #col_name = X.columns.to_series()[np.isnan(X).any()]
3 #col_name
----> 4 clf.fit(X, y)
File ~\anaconda3\lib\site-packages\sklearn\pipeline.py:394, in Pipeline.fit(self, X, y, **fit_params)
392 if self._final_estimator != "passthrough":
393 fit_params_last_step = fit_params_steps[self.steps[-1][0]]
--> 394 self._final_estimator.fit(Xt, y, **fit_params_last_step)
396 return self
File ~\anaconda3\lib\site-packages\sklearn\neural_network\_multilayer_perceptron.py:752, in BaseMultilayerPerceptron.fit(self, X, y)
735 def fit(self, X, y):
736 """Fit the model to data matrix X and target(s) y.
737
738 Parameters
(...)
750 Returns a trained MLP model.
751 """
--> 752 return self._fit(X, y, incremental=False)
File ~\anaconda3\lib\site-packages\sklearn\neural_network\_multilayer_perceptron.py:393, in BaseMultilayerPerceptron._fit(self, X, y, incremental)
386 raise ValueError(
387 "hidden_layer_sizes must be > 0, got %s." % hidden_layer_sizes
388 )
389 first_pass = not hasattr(self, "coefs_") or (
390 not self.warm_start and not incremental
391 )
--> 393 X, y = self._validate_input(X, y, incremental, reset=first_pass)
395 n_samples, n_features = X.shape
397 # Ensure y is 2D
File ~\anaconda3\lib\site-packages\sklearn\neural_network\_multilayer_perceptron.py:1100, in MLPClassifier._validate_input(self, X, y, incremental, reset)
1099 def _validate_input(self, X, y, incremental, reset):
-> 1100 X, y = self._validate_data(
1101 X,
1102 y,
1103 accept_sparse=["csr", "csc"],
1104 multi_output=True,
1105 dtype=(np.float64, np.float32),
1106 reset=reset,
1107 )
1108 if y.ndim == 2 and y.shape[1] == 1:
1109 y = column_or_1d(y, warn=True)
File ~\anaconda3\lib\site-packages\sklearn\base.py:581, in BaseEstimator._validate_data(self, X, y, reset, validate_separately, **check_params)
579 y = check_array(y, **check_y_params)
580 else:
--> 581 X, y = check_X_y(X, y, **check_params)
582 out = X, y
584 if not no_val_X and check_params.get("ensure_2d", True):
File ~\anaconda3\lib\site-packages\sklearn\utils\validation.py:979, in check_X_y(X, y, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, multi_output, ensure_min_samples, ensure_min_features, y_numeric, estimator)
962 raise ValueError("y cannot be None")
964 X = check_array(
965 X,
966 accept_sparse=accept_sparse,
(...)
976 estimator=estimator,
977 )
--> 979 y = _check_y(y, multi_output=multi_output, y_numeric=y_numeric)
981 check_consistent_length(X, y)
983 return X, y
File ~\anaconda3\lib\site-packages\sklearn\utils\validation.py:989, in _check_y(y, multi_output, y_numeric)
987 """Isolated part of check_X_y dedicated to y validation"""
988 if multi_output:
--> 989 y = check_array(
990 y, accept_sparse="csr", force_all_finite=True, ensure_2d=False, dtype=None
991 )
992 else:
993 y = column_or_1d(y, warn=True)
File ~\anaconda3\lib\site-packages\sklearn\utils\validation.py:800, in check_array(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator)
794 raise ValueError(
795 "Found array with dim %d. %s expected <= 2."
796 % (array.ndim, estimator_name)
797 )
799 if force_all_finite:
--> 800 _assert_all_finite(array, allow_nan=force_all_finite == "allow-nan")
802 if ensure_min_samples > 0:
803 n_samples = _num_samples(array)
File ~\anaconda3\lib\site-packages\sklearn\utils\validation.py:114, in _assert_all_finite(X, allow_nan, msg_dtype)
107 if (
108 allow_nan
109 and np.isinf(X).any()
110 or not allow_nan
111 and not np.isfinite(X).all()
112 ):
113 type_err = "infinity" if allow_nan else "NaN, infinity"
--> 114 raise ValueError(
115 msg_err.format(
116 type_err, msg_dtype if msg_dtype is not None else X.dtype
117 )
118 )
119 # for object dtype data, we only check for NaNs (GH-13254)
120 elif X.dtype == np.dtype("object") and not allow_nan:
ValueError: Input contains NaN, infinity or a value too large for dtype('float64').
You may want to check nan as well
col_name = X.columns.to_series()[np.isinf(X).any() | np.isnan(X).any()]
Everything in my code is working well, until I try to use the KNN algorithm to predict the quality of wine using its attributes. This is my first time trying this code for KNN.
this part is giving me errors
from sklearn.neighbors import KNeighborsClassifier
classifier = KNeighborsClassifier(n_neighbors=0)
classifier.fit(wine_train[X], y_train)
Error location:
1 from sklearn.neighbors import KNeighborsClassifier
2 classifier = KNeighborsClassifier(n_neighbors=0)
----> 3 classifier.fit(wine_train[X], y_train)
File /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages/sklearn/utils/validation.py:979, in check_X_y(X, y, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, multi_output, ensure_min_samples, ensure_min_features, y_numeric, estimator)
962 raise ValueError("y cannot be None")
964 X = check_array(
965 X,
966 accept_sparse=accept_sparse,
(...)
976 estimator=estimator,
977 )
--> 979 y = _check_y(y, multi_output=multi_output, y_numeric=y_numeric)
981 check_consistent_length(X, y)
983 return X, y
File /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages/sklearn/utils/validation.py:989, in _check_y(y, multi_output, y_numeric)
987 """Isolated part of check_X_y dedicated to y validation"""
988 if multi_output:
--> 989 y = check_array(
990 y, accept_sparse="csr", force_all_finite=True, ensure_2d=False, dtype=None
991 )
992 else:
993 y = column_or_1d(y, warn=True)
File /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages/sklearn/utils/validation.py:800, in check_array(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator)
794 raise ValueError(
795 "Found array with dim %d. %s expected <= 2."
796 % (array.ndim, estimator_name)
797 )
799 if force_all_finite:
--> 800 _assert_all_finite(array, allow_nan=force_all_finite == "allow-nan")
802 if ensure_min_samples > 0:
803 n_samples = _num_samples(array)
File /Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages/sklearn/utils/validation.py:122, in _assert_all_finite(X, allow_nan, msg_dtype)
120 elif X.dtype == np.dtype("object") and not allow_nan:
121 if _object_dtype_isnan(X).any():
--> 122 raise ValueError("Input contains NaN")
ValueError: Input contains NaN
I want to create a logistic regression model to predict if the relationship is known or unknown, I've set the known values to 1 and unknown to 0 in the dataset. I have also added several features to train the data and predict the relationship.
I have run this code:
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
import sklearn.datasets
from sklearn.model_selection import train_test_split
import pandas as pd
df = pd.read_csv('boo.csv')
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
y = pd.get_dummies(df.Relationship, prefix='Relationship')
X = pd.get_dummies(df, columns=['Relationship', 'Month','Year','Victim Age', 'Perpetrator
Age', 'Victim Sex', 'Victim Race', 'Perpetrator Sex', 'Crime Type', 'Perpetrator Race'], drop_first = True )
X_train, X_test, y_train, y_test = train_test_split(df[['Month','Year','Victim Age', 'Perpetrator Age', 'Victim Sex', 'Victim Race', 'Perpetrator Sex', 'Crime Type', 'Perpetrator Race']], df.Relationship, test_size=0.1)
model = LogisticRegression()
np.isnan(X)
np.where(np.isnan(X))
np.nan_to_num(X)
model.fit(X, y)
I am encountering this error:
ValueError Traceback (most recent call last)
<ipython-input-101-68355fc70ed4> in <module>
15 np.where(np.isnan(X))
16 np.nan_to_num(X)
---> 17 model.fit(X, y)
~\anaconda3\lib\site-packages\sklearn\linear_model\_logistic.py in fit(self, X, y, sample_weight)
1342 _dtype = [np.float64, np.float32]
1343
-> 1344 X, y = self._validate_data(X, y, accept_sparse='csr', dtype=_dtype,
1345 order="C",
1346 accept_large_sparse=solver != 'liblinear')
~\anaconda3\lib\site-packages\sklearn\base.py in _validate_data(self, X, y, reset, validate_separately, **check_params)
431 y = check_array(y, **check_y_params)
432 else:
--> 433 X, y = check_X_y(X, y, **check_params)
434 out = X, y
435
~\anaconda3\lib\site-packages\sklearn\utils\validation.py in inner_f(*args, **kwargs)
61 extra_args = len(args) - len(all_args)
62 if extra_args <= 0:
---> 63 return f(*args, **kwargs)
64
65 # extra_args > 0
~\anaconda3\lib\site-packages\sklearn\utils\validation.py in check_X_y(X, y, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, multi_output, ensure_min_samples, ensure_min_features, y_numeric, estimator)
812 raise ValueError("y cannot be None")
813
--> 814 X = check_array(X, accept_sparse=accept_sparse,
815 accept_large_sparse=accept_large_sparse,
816 dtype=dtype, order=order, copy=copy,
~\anaconda3\lib\site-packages\sklearn\utils\validation.py in inner_f(*args, **kwargs)
61 extra_args = len(args) - len(all_args)
62 if extra_args <= 0:
---> 63 return f(*args, **kwargs)
64
65 # extra_args > 0
~\anaconda3\lib\site-packages\sklearn\utils\validation.py in check_array(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator)
661
662 if force_all_finite:
--> 663 _assert_all_finite(array,
664 allow_nan=force_all_finite == 'allow-nan')
665
~\anaconda3\lib\site-packages\sklearn\utils\validation.py in _assert_all_finite(X, allow_nan, msg_dtype)
101 not allow_nan and not np.isfinite(X).all()):
102 type_err = 'infinity' if allow_nan else 'NaN, infinity'
--> 103 raise ValueError(
104 msg_err.format
105 (type_err,
ValueError: Input contains NaN, infinity or a value too large for dtype('float64').
I viewed several stackoverflow problems and tried their solutions, but nothing seems to work. This error occurs only when I try to fit X and y in the model.
You need to fix the 'nan' values in your dataset. You are checking for the nan values but not fixing it. Below is sample data.
X = np.concatenate((np.arange(1,15),[np.NaN,np.NaN]))
print(np.isnan(X), np.where(np.isnan(X)), np.nan_to_num(X))
Output:
[False False False False False False False False False False False False
False False True True]
(array([14, 15]),)
[ 1. 2. 3. 4. 5. 6. 7. 8. 9. 10. 11. 12. 13. 14. 0. 0.]
you need to do the next step and assign it back. The code doesnt do 'inplace' assigning.if you dont assign it,there wont be changes in the original array.
X = np.nan_to_num(X) # assign it
Now if you check X again, there wont be 'nan' values, and then proceed with training the model.
hope you are having a great day! I know the problem is silly and most of you can probably figure it out. But i do need help. So yeah. Here's the problem:
The code goes something like this:
import pandas as pd
from sklearn.tree import DecisionTreeClassifier
data = pd.read_csv("Data.csv")
X = data.drop(columns = "Answers")
Y = data["Answers"]
algorithm = DecisionTreeClassifier()
algorithm.fit(X, Y)
And I know I'm dumb and probably don't know why its outputting this shi*:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-46-0deefc30b34f> in <module>
9 algorithm = DecisionTreeClassifier()
10
---> 11 algorithm.fit(X, Y)
~\anaconda3\lib\site-packages\sklearn\tree\_classes.py in fit(self, X, y, sample_weight, check_input, X_idx_sorted)
888 """
889
--> 890 super().fit(
891 X, y,
892 sample_weight=sample_weight,
~\anaconda3\lib\site-packages\sklearn\tree\_classes.py in fit(self, X, y, sample_weight, check_input, X_idx_sorted)
154 check_X_params = dict(dtype=DTYPE, accept_sparse="csc")
155 check_y_params = dict(ensure_2d=False, dtype=None)
--> 156 X, y = self._validate_data(X, y,
157 validate_separately=(check_X_params,
158 check_y_params))
~\anaconda3\lib\site-packages\sklearn\base.py in _validate_data(self, X, y, reset, validate_separately, **check_params)
427 # :(
428 check_X_params, check_y_params = validate_separately
--> 429 X = check_array(X, **check_X_params)
430 y = check_array(y, **check_y_params)
431 else:
~\anaconda3\lib\site-packages\sklearn\utils\validation.py in inner_f(*args, **kwargs)
70 FutureWarning)
71 kwargs.update({k: arg for k, arg in zip(sig.parameters, args)})
---> 72 return f(**kwargs)
73 return inner_f
74
~\anaconda3\lib\site-packages\sklearn\utils\validation.py in check_array(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator)
596 array = array.astype(dtype, casting="unsafe", copy=False)
597 else:
--> 598 array = np.asarray(array, order=order, dtype=dtype)
599 except ComplexWarning:
600 raise ValueError("Complex data not supported\n"
~\anaconda3\lib\site-packages\numpy\core\_asarray.py in asarray(a, dtype, order)
81
82 """
---> 83 return array(a, dtype, copy=False, order=order)
84
85
~\anaconda3\lib\site-packages\pandas\core\generic.py in __array__(self, dtype)
1779
1780 def __array__(self, dtype=None) -> np.ndarray:
-> 1781 return np.asarray(self._values, dtype=dtype)
1782
1783 def __array_wrap__(self, result, context=None):
~\anaconda3\lib\site-packages\numpy\core\_asarray.py in asarray(a, dtype, order)
81
82 """
---> 83 return array(a, dtype, copy=False, order=order)
84
85
ValueError: could not convert string to float: "Wat's your name?"
Thank you for viewing this problem and perhaps in solving it. Have a great day!
I have a single independent variable X where each element is a 2-d array with shape (20, 431). The variable X itself is a 2-d numpy array of shape (200,). . How do I pass this to the sklearn.SVM object?
Edit: actual dataframe:
Category Features
0 1 [[-177.08171, -219.89174, -253.55954, -218.560...
1 0 [[-291.89288, -316.40735, -389.8398, -413.6302...
2 1 [[-355.88293, -351.0909, -364.43524, -400.7097.
Each element of Features is a 20*431 numpy array. I need to use these features to classify the category.
x = data.iloc[:, 1].values
y = data.iloc[:, 0].values
x.shape
(200, )
x[0].shape
(20, 431)
y.shape
(200, )
Fitting the model after splitting into train and test data:
classifier = SVC(kernel = 'rbf', random_state=0)
classifier.fit(x_train, y_train)
Error:
TypeError Traceback (most recent call last)
TypeError: only size-1 arrays can be converted to Python scalars
The above exception was the direct cause of the following exception:
ValueError Traceback (most recent call last)
<ipython-input-203-fdf0fc8db087> in <module>
----> 1 classifier.fit(x_train, y_train)
~\Anaconda3\envs\LangDetEnv1.0\lib\site-packages\sklearn\svm\_base.py in fit(self, X, y, sample_weight)
160 X, y = self._validate_data(X, y, dtype=np.float64,
161 order='C', accept_sparse='csr',
--> 162 accept_large_sparse=False)
163
164 y = self._validate_targets(y)
~\Anaconda3\envs\LangDetEnv1.0\lib\site-packages\sklearn\base.py in _validate_data(self, X, y, reset, validate_separately, **check_params)
430 y = check_array(y, **check_y_params)
431 else:
--> 432 X, y = check_X_y(X, y, **check_params)
433 out = X, y
434
~\Anaconda3\envs\LangDetEnv1.0\lib\site-packages\sklearn\utils\validation.py in inner_f(*args, **kwargs)
71 FutureWarning)
72 kwargs.update({k: arg for k, arg in zip(sig.parameters, args)})
---> 73 return f(**kwargs)
74 return inner_f
75
~\Anaconda3\envs\LangDetEnv1.0\lib\site-packages\sklearn\utils\validation.py in check_X_y(X, y, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, multi_output, ensure_min_samples, ensure_min_features, y_numeric, estimator)
801 ensure_min_samples=ensure_min_samples,
802 ensure_min_features=ensure_min_features,
--> 803 estimator=estimator)
804 if multi_output:
805 y = check_array(y, accept_sparse='csr', force_all_finite=True,
~\Anaconda3\envs\LangDetEnv1.0\lib\site-packages\sklearn\utils\validation.py in inner_f(*args, **kwargs)
71 FutureWarning)
72 kwargs.update({k: arg for k, arg in zip(sig.parameters, args)})
---> 73 return f(**kwargs)
74 return inner_f
75
~\Anaconda3\envs\LangDetEnv1.0\lib\site-packages\sklearn\utils\validation.py in check_array(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator)
597 array = array.astype(dtype, casting="unsafe", copy=False)
598 else:
--> 599 array = np.asarray(array, order=order, dtype=dtype)
600 except ComplexWarning:
601 raise ValueError("Complex data not supported\n"
~\Anaconda3\envs\LangDetEnv1.0\lib\site-packages\numpy\core\_asarray.py in asarray(a, dtype, order)
81
82 """
---> 83 return array(a, dtype, copy=False, order=order)
84
85
ValueError: setting an array element with a sequence.