Related
directory = "path/to/directory"
filename = "C:\\Users\\home\\Desktop\\Python Projects\\TelcomCustomer-Churn_2.csv"
full_path = os.path.join(directory, filename)
def load_csv(path='C:\\Users\\home\\Desktop\\Python Projects\\TelcomCustomer-Churn_2.csv'):
df = pd.read_csv('C:\\Users\\home\\Desktop\\Python Projects\\TelcomCustomer-Churn_2.csv')
return df
def preprocess_data(df):
df['TotalCharges'] = df['TotalCharges'].replace(" ", 0).astype('float')
# Identify numeric and categorical columns
numeric_cols = df.select_dtypes(include='number').columns.tolist()
categorical_cols = df.select_dtypes(include='object').columns.tolist()
# Fill in missing numeric values with the mean of the column
for col in numeric_cols:
df[col].fillna(df[col].mean(), inplace=True)
# Fill in missing categorical values with the mode of the column
for col in categorical_cols:
df[col].fillna(df[col].mode()[0], inplace=True)
# Drop duplicates
df.drop_duplicates(inplace=True)
# One-hot encode categorical features
X_cat = pd.get_dummies(df, columns=categorical_cols)
X = X_cat.drop(columns='Churn_Yes', axis=1)
y = X_cat['Churn_Yes']
return X, y
def Split_data(X, y):
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
# Scale the features
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)
return X_train_scaled, X_test_scaled, y_train, y_test
)
print(Split_data(X, y))
the error:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
~\AppData\Local\Temp\ipykernel_592\3782283586.py in <module>
----> 1 print(Split_data(X, y))
~\AppData\Local\Temp\ipykernel_592\18160545.py in Split_data(X, y)
5 # Scale the features
6 scaler = StandardScaler()
----> 7 X_train_scaled = scaler.fit_transform(X_train)
8 X_test_scaled = scaler.transform(X_test)
9 return X_train_scaled, X_test_scaled, y_train, y_test
~\anaconda3\lib\site-packages\sklearn\base.py in fit_transform(self, X, y, **fit_params)
850 if y is None:
851 # fit method of arity 1 (unsupervised transformation)
--> 852 return self.fit(X, **fit_params).transform(X)
853 else:
854 # fit method of arity 2 (supervised transformation)
~\anaconda3\lib\site-packages\sklearn\preprocessing\_data.py in fit(self, X, y, sample_weight)
804 # Reset internal state before fitting
805 self._reset()
--> 806 return self.partial_fit(X, y, sample_weight)
807
808 def partial_fit(self, X, y=None, sample_weight=None):
~\anaconda3\lib\site-packages\sklearn\preprocessing\_data.py in partial_fit(self, X, y, sample_weight)
839 """
840 first_call = not hasattr(self, "n_samples_seen_")
--> 841 X = self._validate_data(
842 X,
843 accept_sparse=("csr", "csc"),
~\anaconda3\lib\site-packages\sklearn\base.py in _validate_data(self, X, y, reset, validate_separately, **check_params)
564 raise ValueError("Validation should be done on X, y or both.")
565 elif not no_val_X and no_val_y:
--> 566 X = check_array(X, **check_params)
567 out = X
568 elif no_val_X and not no_val_y:
~\anaconda3\lib\site-packages\sklearn\utils\validation.py in check_array(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator)
744 array = array.astype(dtype, casting="unsafe", copy=False)
745 else:
--> 746 array = np.asarray(array, order=order, dtype=dtype)
747 except ComplexWarning as complex_warning:
748 raise ValueError(
~\anaconda3\lib\site-packages\pandas\core\generic.py in __array__(self, dtype)
2062
2063 def __array__(self, dtype: npt.DTypeLike | None = None) -> np.ndarray:
-> 2064 return np.asarray(self._values, dtype=dtype)
2065
2066 def __array_wrap__(
ValueError: could not convert string to float: '4223-BKEOR'
I did not understand where i have done the mistake in the above code
The error happend because one of the data in your X_train contains non numeric value.
Try the below and see if it works:
numeric_cols = X_train.select_dtypes(include='number').columns.tolist()
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train[numeric_cols])
X_test_scaled = scaler.transform(X_test[numeric_cols])
It happens beacause,your dataset may contains some of the column as object type, as python by default allocated memory in heap. So check the datatypes of column in dataframe by dataset_Name.dtypes and make sure to typecast it into float64.
for our project work in university we need to write a machine learning code. Unfortunately I don't have any programming knowledge and am a bit helpless.
My classification is Naive Bayes and when I run the code I get an error:
ValueError: Input contains NaN, infinity or a value too large for dtype('float64').
Unfortunately, I can't do anything with this and also nothing with the solutions found so far in the forum. Maybe someone can help me?
dat = pd.get_dummies(df)
# Define X and y
X = dat.drop('RESP', axis = 1)
y = dat['RESP']
# training and testing data
from sklearn.model_selection import train_test_split
# assign test data size 30%
X_train, X_test, y_train, y_test =train_test_split(X,y,test_size= 0.3, random_state=0)
#BERNOULLI
# importing classifier
from sklearn.naive_bayes import BernoulliNB
# initializaing the NB
classifer = BernoulliNB()
# training the model
classifer.fit(X_train, y_train)
# testing the model
y_pred = classifer.predict(X_test)
Error:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-22-df3c037d90d0> in <module>
23
24 # testing the model
---> 25 y_pred = classifer.predict(X_test)
5 frames
/usr/local/lib/python3.8/dist-packages/sklearn/naive_bayes.py in predict(self, X)
80 """
81 check_is_fitted(self)
---> 82 X = self._check_X(X)
83 jll = self._joint_log_likelihood(X)
84 return self.classes_[np.argmax(jll, axis=1)]
/usr/local/lib/python3.8/dist-packages/sklearn/naive_bayes.py in _check_X(self, X)
1145 def _check_X(self, X):
1146 """Validate X, used only in predict* methods."""
-> 1147 X = super()._check_X(X)
1148 if self.binarize is not None:
1149 X = binarize(X, threshold=self.binarize)
/usr/local/lib/python3.8/dist-packages/sklearn/naive_bayes.py in _check_X(self, X)
517 def _check_X(self, X):
518 """Validate X, used only in predict* methods."""
--> 519 return self._validate_data(X, accept_sparse="csr", reset=False)
520
521 def _check_X_y(self, X, y, reset=True):
/usr/local/lib/python3.8/dist-packages/sklearn/base.py in _validate_data(self, X, y, reset, validate_separately, **check_params)
564 raise ValueError("Validation should be done on X, y or both.")
565 elif not no_val_X and no_val_y:
--> 566 X = check_array(X, **check_params)
567 out = X
568 elif no_val_X and not no_val_y:
/usr/local/lib/python3.8/dist-packages/sklearn/utils/validation.py in check_array(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator)
798
799 if force_all_finite:
--> 800 _assert_all_finite(array, allow_nan=force_all_finite == "allow-nan")
801
802 if ensure_min_samples > 0:
/usr/local/lib/python3.8/dist-packages/sklearn/utils/validation.py in _assert_all_finite(X, allow_nan, msg_dtype)
112 ):
113 type_err = "infinity" if allow_nan else "NaN, infinity"
--> 114 raise ValueError(
115 msg_err.format(
116 type_err, msg_dtype if msg_dtype is not None else X.dtype
ValueError: Input contains NaN, infinity or a value too large for dtype('float64').
I am trying to make a ML model , but I am having problems with this one feature. The error given is saying , cannot convert string to float. I tried using a convert method but it is still not working. This code tries to make a ml model
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sn
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
crime_data= pd.read_csv('Crime_Data_from_2020_to_Present.csv')
#offense= crime_data['Crm Cd Desc']
myData= crime_data.drop(columns=['DR_NO','Date Rptd','Rpt Dist No','Part 1-2','Crm Cd','Mocodes','Vict Age','Vict Sex','Vict Descent','Premis Desc','Weapon Used Cd','Weapon Desc','Status','Status Desc','Crm Cd 1','Crm Cd 2','Crm Cd 3','Crm Cd 4','Cross Street','Premis Cd'])
myData['DATE OCC'] = myData['DATE OCC'].astype(float)#method for converting not working
X= myData.drop (columns=['AREA NAME']) #input data
y= myData['AREA NAME'] #output data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2)
classifier = RandomForestClassifier(n_estimators = 50)
classifier.fit(X_train, y_train)
y_pred = classifier.predict(X_test)
y_pred
This is the error I'm getting
ValueError Traceback (most recent call last)
<ipython-input-16-20d49933ca7e> in <module>
9 #offense= crime_data['Crm Cd Desc']
10 myData= crime_data.drop(columns=['DR_NO','Date Rptd','Rpt Dist No','Part 1-2','Crm Cd','Mocodes','Vict Age','Vict Sex','Vict Descent','Premis Desc','Weapon Used Cd','Weapon Desc','Status','Status Desc','Crm Cd 1','Crm Cd 2','Crm Cd 3','Crm Cd 4','Cross Street','Premis Cd'])
---> 11 myData['DATE OCC'] = myData['DATE OCC'].astype(float)#method for converting not working
12
13 X= myData.drop (columns=['AREA NAME']) #input data
~\anaconda3\lib\site-packages\pandas\core\generic.py in astype(self, dtype, copy, errors)
5696 else:
5697 # else, only a single dtype is given
-> 5698 new_data = self._data.astype(dtype=dtype, copy=copy, errors=errors)
5699 return self._constructor(new_data).__finalize__(self)
5700
~\anaconda3\lib\site-packages\pandas\core\internals\managers.py in astype(self, dtype, copy, errors)
580
581 def astype(self, dtype, copy: bool = False, errors: str = "raise"):
--> 582 return self.apply("astype", dtype=dtype, copy=copy, errors=errors)
583
584 def convert(self, **kwargs):
~\anaconda3\lib\site-packages\pandas\core\internals\managers.py in apply(self, f, filter, **kwargs)
440 applied = b.apply(f, **kwargs)
441 else:
--> 442 applied = getattr(b, f)(**kwargs)
443 result_blocks = _extend_blocks(applied, result_blocks)
444
~\anaconda3\lib\site-packages\pandas\core\internals\blocks.py in astype(self, dtype, copy, errors)
623 vals1d = values.ravel()
624 try:
--> 625 values = astype_nansafe(vals1d, dtype, copy=True)
626 except (ValueError, TypeError):
627 # e.g. astype_nansafe can fail on object-dtype of strings
~\anaconda3\lib\site-packages\pandas\core\dtypes\cast.py in astype_nansafe(arr, dtype, copy, skipna)
895 if copy or is_object_dtype(arr) or is_object_dtype(dtype):
896 # Explicit copy, or required since NumPy can't view from / to object.
--> 897 return arr.astype(dtype, copy=True)
898
899 return arr.view(dtype)
ValueError: could not convert string to float: '01/08/2020 12:00:00 AM'
I changed the code to this
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sn
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
crime_data= pd.read_csv('Crime_Data_from_2020_to_Present.csv')
#offense= crime_data['Crm Cd Desc']
myData= crime_data.drop(columns=['DR_NO','Date Rptd','Rpt Dist No','Part 1-2','Crm Cd','Mocodes','Vict Age','Vict Sex','Vict Descent','Premis Desc','Weapon Used Cd','Weapon Desc','Status','Status Desc','Crm Cd 1','Crm Cd 2','Crm Cd 3','Crm Cd 4','Cross Street','Premis Cd'])
myData['DATE OCC'] = pd.to_datetime(myData['DATE OCC'])#method for converting not working
X= myData.drop (columns=['AREA NAME']) #input data
y= myData['AREA NAME'] #output data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2)
classifier = RandomForestClassifier(n_estimators = 50)
classifier.fit(X_train, y_train)
y_pred = classifier.predict(X_test)
y_pred
But then I get this error. Please help
TypeError Traceback (most recent call last)
<ipython-input-17-18d34976fcb7> in <module>
16 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2)
17 classifier = RandomForestClassifier(n_estimators = 50)
---> 18 classifier.fit(X_train, y_train)
19 y_pred = classifier.predict(X_test)
20 y_pred
~\anaconda3\lib\site-packages\sklearn\ensemble\_forest.py in fit(self, X, y, sample_weight)
301 "sparse multilabel-indicator for y is not supported."
302 )
--> 303 X, y = self._validate_data(X, y, multi_output=True,
304 accept_sparse="csc", dtype=DTYPE)
305 if sample_weight is not None:
~\anaconda3\lib\site-packages\sklearn\base.py in _validate_data(self, X, y, reset, validate_separately, **check_params)
430 y = check_array(y, **check_y_params)
431 else:
--> 432 X, y = check_X_y(X, y, **check_params)
433 out = X, y
434
~\anaconda3\lib\site-packages\sklearn\utils\validation.py in inner_f(*args, **kwargs)
71 FutureWarning)
72 kwargs.update({k: arg for k, arg in zip(sig.parameters, args)})
---> 73 return f(**kwargs)
74 return inner_f
75
~\anaconda3\lib\site-packages\sklearn\utils\validation.py in check_X_y(X, y, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, multi_output, ensure_min_samples, ensure_min_features, y_numeric, estimator)
794 raise ValueError("y cannot be None")
795
--> 796 X = check_array(X, accept_sparse=accept_sparse,
797 accept_large_sparse=accept_large_sparse,
798 dtype=dtype, order=order, copy=copy,
~\anaconda3\lib\site-packages\sklearn\utils\validation.py in inner_f(*args, **kwargs)
71 FutureWarning)
72 kwargs.update({k: arg for k, arg in zip(sig.parameters, args)})
---> 73 return f(**kwargs)
74 return inner_f
75
~\anaconda3\lib\site-packages\sklearn\utils\validation.py in check_array(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator)enter code here
532
533 if all(isinstance(dtype, np.dtype) for dtype in dtypes_orig):
--> 534 dtype_orig = np.result_type(*dtypes_orig)
535
536 if dtype_numeric:
<__array_function__ internals> in result_type(*args, **kwargs)
TypeError: invalid type promotion
There is a column named DATE OCC in your dataset which has date and time mentioned in it. You are getting that error because, your model is expecting float values and that DATE OCC column you have is in object or datetime64 format. So you have to add this code that I mentioned below:
myData['DATE OCC'] = myData['DATE OCC'].astype('datetime64')
myData['day'] = myData['DATE OCC'].dt.day
myData['month'] = myData['DATE OCC'].dt.month
myData['Year'] = myData['DATE OCC'].dt.year
del myData['DATE OCC']
myData = pd.get_dummies(myData)
X= myData.drop(columns=['AREA NAME']) #input data
y= myData['AREA NAME'] #output data
I'm trying to run a Classification Tree on data about Visa applications (includes categories such as workplace, average pay, etc.) but I'm pulling the following error:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-114-33e98dd52d68> in <module>
3 from sklearn.model_selection import cross_val_predict
4 clf = tree.DecisionTreeClassifier()
----> 5 y_pred = cross_val_predict(clf, x, y, cv=10)
6 cm = ConfusionMatrix(y, y_pred)
7 print(cm)
/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py in cross_val_predict(estimator, X, y, groups, cv, n_jobs, verbose, fit_params, pre_dispatch, method)
775 prediction_blocks = parallel(delayed(_fit_and_predict)(
776 clone(estimator), X, y, train, test, verbose, fit_params, method)
--> 777 for train, test in cv.split(X, y, groups))
778
779 # Concatenate the predictions
/anaconda3/lib/python3.7/site-packages/sklearn/externals/joblib/parallel.py in __call__(self, iterable)
915 # remaining jobs.
916 self._iterating = False
--> 917 if self.dispatch_one_batch(iterator):
918 self._iterating = self._original_iterator is not None
919
/anaconda3/lib/python3.7/site-packages/sklearn/externals/joblib/parallel.py in dispatch_one_batch(self, iterator)
757 return False
758 else:
--> 759 self._dispatch(tasks)
760 return True
761
/anaconda3/lib/python3.7/site-packages/sklearn/externals/joblib/parallel.py in _dispatch(self, batch)
714 with self._lock:
715 job_idx = len(self._jobs)
--> 716 job = self._backend.apply_async(batch, callback=cb)
717 # A job can complete so quickly than its callback is
718 # called before we get here, causing self._jobs to
/anaconda3/lib/python3.7/site-packages/sklearn/externals/joblib/_parallel_backends.py in apply_async(self, func, callback)
180 def apply_async(self, func, callback=None):
181 """Schedule a func to be run"""
--> 182 result = ImmediateResult(func)
183 if callback:
184 callback(result)
/anaconda3/lib/python3.7/site-packages/sklearn/externals/joblib/_parallel_backends.py in __init__(self, batch)
547 # Don't delay the application, to avoid keeping the input
548 # arguments in memory
--> 549 self.results = batch()
550
551 def get(self):
/anaconda3/lib/python3.7/site-packages/sklearn/externals/joblib/parallel.py in __call__(self)
223 with parallel_backend(self._backend, n_jobs=self._n_jobs):
224 return [func(*args, **kwargs)
--> 225 for func, args, kwargs in self.items]
226
227 def __len__(self):
/anaconda3/lib/python3.7/site-packages/sklearn/externals/joblib/parallel.py in <listcomp>(.0)
223 with parallel_backend(self._backend, n_jobs=self._n_jobs):
224 return [func(*args, **kwargs)
--> 225 for func, args, kwargs in self.items]
226
227 def __len__(self):
/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_validation.py in _fit_and_predict(estimator, X, y, train, test, verbose, fit_params, method)
848 estimator.fit(X_train, **fit_params)
849 else:
--> 850 estimator.fit(X_train, y_train, **fit_params)
851 func = getattr(estimator, method)
852 predictions = func(X_test)
/anaconda3/lib/python3.7/site-packages/sklearn/tree/tree.py in fit(self, X, y, sample_weight, check_input, X_idx_sorted)
799 sample_weight=sample_weight,
800 check_input=check_input,
--> 801 X_idx_sorted=X_idx_sorted)
802 return self
803
/anaconda3/lib/python3.7/site-packages/sklearn/tree/tree.py in fit(self, X, y, sample_weight, check_input, X_idx_sorted)
114 random_state = check_random_state(self.random_state)
115 if check_input:
--> 116 X = check_array(X, dtype=DTYPE, accept_sparse="csc")
117 y = check_array(y, ensure_2d=False, dtype=None)
118 if issparse(X):
/anaconda3/lib/python3.7/site-packages/sklearn/utils/validation.py in check_array(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, warn_on_dtype, estimator)
525 try:
526 warnings.simplefilter('error', ComplexWarning)
--> 527 array = np.asarray(array, dtype=dtype, order=order)
528 except ComplexWarning:
529 raise ValueError("Complex data not supported\n"
/anaconda3/lib/python3.7/site-packages/numpy/core/numeric.py in asarray(a, dtype, order)
499
500 """
--> 501 return array(a, dtype, copy=False, order=order)
502
503
TypeError: float() argument must be a string or a number, not 'Timestamp'
I've tried resetting the Kernel but I'm confused on what the error is telling me
The code I am trying to run is this:
from sklearn import tree
from pandas_ml import ConfusionMatrix
from sklearn.model_selection import cross_val_predict
clf = tree.DecisionTreeClassifier()
y_pred = cross_val_predict(clf, x, y, cv=10)
cm = ConfusionMatrix(y, y_pred)
print(cm)
cm.print_stats()
It should be producing the stats when ran correctly
im trying to create a simple classification with tree classifier for disease symptoms. i have tried it using sklearn tree classifier.
it gives the following error. both my code and error is there.
Any suggestion ?
import numpy as np
from sklearn import tree
symptoms = [['flat face','poor moro','hypotonia'],['small head','small jaw','overlapping fingers'], ['small eyes','cleft lip','cleft palate']]
lables = [['Trisomy 21'],['Trisomy 18'],['Trisomy 13']]
classify = tree.DecisionTreeClassifier()
classify = classify.fit(symptoms, lables)
it gives the following error
ValueError Traceback (most recent call last)
<ipython-input-25-0f2c956618c2> in <module>
4 lables = [['Trisomy 21'],['Trisomy 18'],['Trisomy 13']]
5 classify = tree.DecisionTreeClassifier()
----> 6 classify = classify.fit(symptoms, lables)
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\sklearn\tree\tree.py in fit(self, X, y, sample_weight, check_input, X_idx_sorted)
799 sample_weight=sample_weight,
800 check_input=check_input,
--> 801 X_idx_sorted=X_idx_sorted)
802 return self
803
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\sklearn\tree\tree.py in fit(self, X, y, sample_weight, check_input, X_idx_sorted)
114 random_state = check_random_state(self.random_state)
115 if check_input:
--> 116 X = check_array(X, dtype=DTYPE, accept_sparse="csc")
117 y = check_array(y, ensure_2d=False, dtype=None)
118 if issparse(X):
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\sklearn\utils\validation.py in check_array(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, warn_on_dtype, estimator)
525 try:
526 warnings.simplefilter('error', ComplexWarning)
--> 527 array = np.asarray(array, dtype=dtype, order=order)
528 except ComplexWarning:
529 raise ValueError("Complex data not supported\n"
c:\users\admin\appdata\local\programs\python\python36\lib\site-packages\numpy\core\numeric.py in asarray(a, dtype, order)
499
500 """
--> 501 return array(a, dtype, copy=False, order=order)
502
503
ValueError: could not convert string to float: 'flat face'
You need to use label encoder for encoding your string values. The following will work for your requirement:
import numpy as np
from sklearn import tree
from sklearn.preprocessing import LabelEncoder
symptoms = [['flat face','poor moro','hypotonia'],['small head','small jaw','overlapping fingers'], ['small eyes','cleft lip','cleft palate']]
lables = [['Trisomy 21'],['Trisomy 18'],['Trisomy 13']]
df = pd.concat([pd.DataFrame(symptoms), pd.DataFrame(lables)], axis=1)
x_cols = ['sym1', 'sym2', 'sym3']
y_col = 'target'
df.columns = x_cols + [y_col]
df = df.apply(LabelEncoder().fit_transform)
classify = tree.DecisionTreeClassifier()
classify.fit(df[x_cols].values, df[y_col].values)