Develop Reference

error in loading a custom class in sklearn pipeline using joblib

I am trying to deploy an sklearn pipeline using FastApi
so first i saved my pipeline in a job lib file.
My pipeline looks like this:
pipe = Pipeline([('encoder', MultiColumnLabelEncoder(columns =['sg', 'al', 'su', 'rbc', 'pc', 'pcc', 'ba', 'htn', 'dm', 'cad', 'appet','pe', 'ane'] )),
('scaler', StandardScaler()),
('model', lgb_model.best_estimator_)])
and my custom label encoder class looks like this:
class MultiColumnLabelEncoder:
def __init__(self,columns = None):
self.columns = columns # array of column names to encode
def fit(self,X,y=None):
return self # not relevant here
def transform(self,X):
'''
Transforms columns of X specified in self.columns using
LabelEncoder(). If no columns specified, transforms all
columns in X.
'''
output = X.copy()
if self.columns is not None:
for col in self.columns:
output[col] = LabelEncoder().fit_transform(output[col])
else:
for colname,col in output.iteritems():
output[colname] = LabelEncoder().fit_transform(col)
return output
def fit_transform(self,X,y=None):
return self.fit(X,y).transform(X)
The columns attribute in the instantiation is for specifying the categorical variables then i tried loading it in my web app using the code below.
import models.ml.classifier as clf
from fastapi import FastAPI
from joblib import load
from models.data import data_input
import pandas as pd
from utils import MultiColumnLabelEncoder
app = FastAPI(title="deployment",
description="API for machine learning project",
version="1.0")
#app.on_event('startup')
def load_model():
clf.model = load('models/ml/ckd_model.joblib')
I am getting the error below:
raise AttributeError("Can't get attribute {!r} on {!r}" attributeError: Can't get attribute 'MultiColumnLabelEncoder' on <module ' main ' from 'C:\Users\User\anaconda3\Scripts\uvicorn.exe\ main .ple>
[31mERROR.q0m: Application startup failed. Exiting.

Try importing the "main" module from anaconda home directory.

Trying to run Flask and getting message saying 'ModuleNotFoundError: No module named 'sklearn''

I have two python files. Here is my ml.py file.
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn.linear_model import LinearRegression
import pickle
data = pd.read_csv("C:\\Users\\ryans\\Desktop\\insurance.csv")
le = LabelEncoder()
le.fit(data['sex'])
data['Sex'] = le.transform(data['sex'])
le.fit(data['smoker'])
data['Smoker'] = le.transform(data['smoker'])
le.fit(data['region'])
data['Region'] = le.transform(data['region'])
#independent and dependent columns
x = data[["age", "bmi", "children", "Sex", "Smoker", "Region"]]
y = data['charges']
#split in train and test
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.3, random_state=0)
#model training
linreg = LinearRegression()
linreg.fit(x_train, y_train)
#model testing
predictions = linreg.predict(x_test)
linreg.score(x_test,y_test)
#save the model
file = open("C:\\Users\\ryans\\Desktop\\expense_model.pkl", 'wb')
pickle.dump(linreg, file)
Here is my run.py file.
from flask import Flask, render_template, request
import pickle
app = Flask(__name__)
model = pickle.load(open('C:\\Users\\ryans\\Desktop\\expense_model.pkl','rb')) #read mode
#app.route("/")
def home():
return render_template('index.html')
#app.route("/predict", methods=['GET','POST'])
def predict():
if request.method == 'POST':
#access the data from form
## Age
age = int(request.form["age"])
bmi = int(request.form["bmi"])
children = int(request.form["children"])
Sex = int(request.form["Sex"])
Smoker = int(request.form["Smoker"])
Region = int(request.form["Region"])
#get prediction
input_cols = [[age, bmi, children, Sex, Smoker, Region]]
prediction = model.predict(input_cols)
output = round(prediction[0], 2)
return render_template("index.html", prediction_text='Your predicted annual Healthcare Expense is $ {}'.format(output))
if __name__ == "__main__":
app.run(debug=True)
Now, when I try to run run.py in my Anaconda Prompt, I get this error message.
(base) C:\Users\ryans\Desktop\run.py
Traceback (most recent call last):
File "C:\Users\ryans\Desktop\run.py", line 5, in <module>
model = pickle.load(open('C:\\Users\\ryans\\Desktop\\expense_model.pkl','rb')) #read mode
ModuleNotFoundError: No module named 'sklearn'
Sklearn is installed and it works fine when I run it in Spyder. I think, somehow, Flask is not finding the Sklearn correctly. Or, something isn't exposed. Or, rights are not set right. I don't know what's going on with Flask. This is my first time using it. Sklearn is working fine. I know that. Any thoughts on what's wrong here?

You haven't imported sklearn in run.py, so it can't unpickle the sklearn objects in your file.

flask _pickle.PicklingError:

I'm new in flask, I'm trying to implement my text (Bag of word) classifier model in python & deploy it with flask web application. but i got the error when proceeding to other pages by following code:
Implement by using naive bayes classifier, it will render it on news page which gives a result of pos or neg.
from flask import Flask,render_template,url_for,request
import pandas as pd
import pickle
from sklearn.externals import joblib
import matplotlib.pyplot as plt
import csv
from textblob import TextBlob
import sklearn
import _pickle as cPickle
import numpy as np
from scipy.sparse.csr import csr_matrix
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer, TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.svm import SVC, LinearSVC
from sklearn.metrics import classification_report, f1_score, accuracy_score, confusion_matrix
from sklearn.pipeline import Pipeline
from sklearn.model_selection import StratifiedKFold, cross_val_score, train_test_split,GridSearchCV,learning_curve
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import StratifiedKFold
from sklearn.externals import joblib
app = Flask(__name__)
#app.route('/')
def home():
return render_template('home.html')
#app.route('/predict',methods=['POST'])
def predict():
messages = pd.read_csv('bitcoin_reddit.csv', usecols=["title","class"])
messages['length'] = messages['title'].map(lambda text: len(text))
def split_into_tokens(title):
return TextBlob(title).words
def split_into_lemmas(title):
words = TextBlob(title).words.lower()
# for each word, take its "base form" = lemma
return [word.lemma for word in words]
bow_transformer = CountVectorizer(analyzer=split_into_lemmas).fit(messages['title'])
messages_bow = bow_transformer.transform(messages['title'])
tfidf_transformer = TfidfTransformer().fit(messages_bow) #normalization can be done with TF-IDF
messages_tfidf = tfidf_transformer.transform(messages_bow)
#Training Model NB
spam_detector = MultinomialNB().fit(messages_tfidf, messages['class'])
all_predictions = spam_detector.predict(messages_tfidf)
msg_train, msg_test, label_train, label_test = \
train_test_split(messages['title'], messages['class'], test_size=0.2)
pipeline = Pipeline([
('bow', CountVectorizer(analyzer=split_into_lemmas)), # strings to token integer counts
('tfidf', TfidfTransformer()), # integer counts to weighted TF-IDF scores
('classifier', MultinomialNB())]) # train on TF-IDF vectors w/ Naive Bayes classifier
scores = cross_val_score(pipeline, # steps to convert raw messages into models
msg_train, # training data
label_train, # training labels
cv=10, # split data randomly into 10 parts: 9 for training, 1 for scoring
scoring='accuracy') # which scoring metric?
params = {
'tfidf__use_idf': (True, False),
'bow__analyzer': (split_into_lemmas, split_into_tokens),
}
grid = GridSearchCV(
pipeline, # pipeline from above
params, # parameters to tune via cross validation
refit=True, # fit using all available data at the end, on the best found param combination
scoring='accuracy', # what score are we optimizing?
cv=StratifiedKFold(n_splits=5)) # what type of cross validation to use
nb_detector = grid.fit(msg_train, label_train)
predictions = nb_detector.predict(msg_test)
joblib.dump(nb_detector, 'NB_model_bow.pkl')
if request.method == 'POST':
message = request.form['message']
data = [message]
vect = cv.transform(data).toarray()
my_prediction = nb_detector.predict(vect)
return render_template('result.html',prediction = my_prediction)
if __name__ == '__main__':
app.run(debug=True)
But i got this kind of error
_pickle.PicklingError: Can't pickle <function predict.<locals>.split_into_lemmas at 0x000001ABF618AE18>: it's not found as __main__.predict.<locals>.split_into_lemmas

pyspark-2.3 sparkml LogisticRegression model load issue

I am doing a sample pyspark ml exercise where I need to store a model and read it back. I am able to successfully save the model, but when I am trying to read/load it back it is throwing below exception. I am new to spark ml and python, please guide me on this.
Code:
from pyspark.sql import *
from pyspark.ml.feature import RFormula
from pyspark.ml.classification import LogisticRegression
from pyspark.ml import Pipeline
from pyspark.ml.tuning import ParamGridBuilder
from pyspark.ml.evaluation import BinaryClassificationEvaluator
from pyspark.ml.tuning import TrainValidationSplit
from pyspark.ml.tuning import TrainValidationSplitModel
spark = SparkSession.builder.appName("LocalMLSparkSession").master("local").getOrCreate()
df = spark.read.json("/to_data/simpleml.json").orderBy("value2")
df.select(df.color).distinct().show(10, False)
train, test = df.randomSplit([0.7, 0.3])
rForm = RFormula()
ls = LogisticRegression().setLabelCol("label").setFeaturesCol("features")
# setting pipeline
stages = [rForm,ls]
pipeline = Pipeline().setStages(stages)
#setting param grid builder
params = ParamGridBuilder()\
.addGrid(rForm.formula,["lab ~ . + color:value1", "lab ~ . + color:value1 + color:value2"])\
.addGrid(ls.elasticNetParam, [0.0, 0.5, 1.0])\
.addGrid(ls.regParam,[0.1, 0.2])\
.build()
#setting evaluator
evaluator = BinaryClassificationEvaluator()\
.setMetricName("areaUnderROC")\
.setRawPredictionCol("prediction")\
.setLabelCol("label")
#checking hyperparameters to train datasets
tvs = TrainValidationSplit()\
.setTrainRatio(0.75)\
.setEstimatorParamMaps(params)\
.setEstimator(pipeline)\
.setEvaluator(evaluator)
tvsFitted = tvs.fit(train)
evl = evaluator.evaluate(tvsFitted.transform(test))
tvsFitted.transform(test).select("features", "label", "prediction").show(10,False)
print(evl)
pip_model = tvsFitted.bestModel
pip_model.write().overwrite().save("/to_path/sparkml/model")
model = TrainValidationSplitModel().load("/to_path/sparkml/model")
model.transform(test)
Exception:
Traceback (most recent call last):
File "/home/dd/dd/python-workspace/SparkMLPipelineDemo.py", line 59, in <module>
model = TrainValidationSplitModel().load("/to_path/sparkml/model")
TypeError: __init__() missing 1 required positional argument: 'bestModel'
Process finished with exit code 1

You need to remove parenthesis when you load, e.g. replace :
model = TrainValidationSplitModel().load("/to_path/sparkml/model")
with
model = TrainValidationSplitModel.load("/to_path/sparkml/model")

Boost.Python.ArgumentError: Python argument types in ~did not match C++ signature, How can I handle it?

I want to predict Knoell data using ppar_ligand_model.
But some SMILES display this error message.
How can I handle it? Is this problem of SMILE's structure?
And What code should I add to exclude the material's SMILE that causes the error?
Here is soruce code and github link
run_pparg_ligand_model.py
import numpy
import pandas
import sys
from keras.models import Sequential
from keras.layers import Dense
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import StratifiedKFold
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.metrics import confusion_matrix
from rdkit.Chem import AllChem
from rdkit import DataStructs
from rdkit.Chem import Draw
from rdkit.Chem.Draw import SimilarityMaps
from rdkit import Chem, DataStructs
import seaborn as sns
import matplotlib as mpl
import matplotlib.pyplot as plt
from keras.models import load_model
def main():
"""Run the main function."""
outfile = open(sys.argv[1] + "_predictions.txt", 'w')
outfile.write("Chemical Name\tPrediction\n")
model = load_model("pparg_ligand_model.h5")
dataframe = pandas.read_csv(sys.argv[1], sep="\t")
mols = []
fps = []
for index, row in dataframe.iterrows():
# try :
# mol = Chem.MolFromSmiles(row['SMILES'])
# break
# except :
# continue
mol = Chem.MolFromSmiles(row['SMILES'])
fp = AllChem.GetMorganFingerprintAsBitVect(mol, 2)
mols.append(mol)
fps.append(fp)
np_fps = []
for fp in fps:
arr = numpy.zeros((1,))
DataStructs.ConvertToNumpyArray(fp, arr)
np_fps.append(arr)
np_fps_array = numpy.array(np_fps)
predictions = model.predict(np_fps_array, batch_size=5)
i = 0
for prediction in predictions:
y_prediction = ''
if(prediction < 0.50):
y_prediction = "ligand"
else:
y_prediction = "not_ligand"
outfile.write(dataframe['Chemical_Name'][i] + "\t" + y_prediction + "\n")
i += 1
outfile.close()
if __name__ == "__main__":
main()