I'm new in flask, I'm trying to implement my text (Bag of word) classifier model in python & deploy it with flask web application. but i got the error when proceeding to other pages by following code:
Implement by using naive bayes classifier, it will render it on news page which gives a result of pos or neg.
from flask import Flask,render_template,url_for,request
import pandas as pd
import pickle
from sklearn.externals import joblib
import matplotlib.pyplot as plt
import csv
from textblob import TextBlob
import sklearn
import _pickle as cPickle
import numpy as np
from scipy.sparse.csr import csr_matrix
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer, TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.svm import SVC, LinearSVC
from sklearn.metrics import classification_report, f1_score, accuracy_score, confusion_matrix
from sklearn.pipeline import Pipeline
from sklearn.model_selection import StratifiedKFold, cross_val_score, train_test_split,GridSearchCV,learning_curve
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import StratifiedKFold
from sklearn.externals import joblib
app = Flask(__name__)
#app.route('/')
def home():
return render_template('home.html')
#app.route('/predict',methods=['POST'])
def predict():
messages = pd.read_csv('bitcoin_reddit.csv', usecols=["title","class"])
messages['length'] = messages['title'].map(lambda text: len(text))
def split_into_tokens(title):
return TextBlob(title).words
def split_into_lemmas(title):
words = TextBlob(title).words.lower()
# for each word, take its "base form" = lemma
return [word.lemma for word in words]
bow_transformer = CountVectorizer(analyzer=split_into_lemmas).fit(messages['title'])
messages_bow = bow_transformer.transform(messages['title'])
tfidf_transformer = TfidfTransformer().fit(messages_bow) #normalization can be done with TF-IDF
messages_tfidf = tfidf_transformer.transform(messages_bow)
#Training Model NB
spam_detector = MultinomialNB().fit(messages_tfidf, messages['class'])
all_predictions = spam_detector.predict(messages_tfidf)
msg_train, msg_test, label_train, label_test = \
train_test_split(messages['title'], messages['class'], test_size=0.2)
pipeline = Pipeline([
('bow', CountVectorizer(analyzer=split_into_lemmas)), # strings to token integer counts
('tfidf', TfidfTransformer()), # integer counts to weighted TF-IDF scores
('classifier', MultinomialNB())]) # train on TF-IDF vectors w/ Naive Bayes classifier
scores = cross_val_score(pipeline, # steps to convert raw messages into models
msg_train, # training data
label_train, # training labels
cv=10, # split data randomly into 10 parts: 9 for training, 1 for scoring
scoring='accuracy') # which scoring metric?
params = {
'tfidf__use_idf': (True, False),
'bow__analyzer': (split_into_lemmas, split_into_tokens),
}
grid = GridSearchCV(
pipeline, # pipeline from above
params, # parameters to tune via cross validation
refit=True, # fit using all available data at the end, on the best found param combination
scoring='accuracy', # what score are we optimizing?
cv=StratifiedKFold(n_splits=5)) # what type of cross validation to use
nb_detector = grid.fit(msg_train, label_train)
predictions = nb_detector.predict(msg_test)
joblib.dump(nb_detector, 'NB_model_bow.pkl')
if request.method == 'POST':
message = request.form['message']
data = [message]
vect = cv.transform(data).toarray()
my_prediction = nb_detector.predict(vect)
return render_template('result.html',prediction = my_prediction)
if __name__ == '__main__':
app.run(debug=True)
But i got this kind of error
_pickle.PicklingError: Can't pickle <function predict.<locals>.split_into_lemmas at 0x000001ABF618AE18>: it's not found as __main__.predict.<locals>.split_into_lemmas
Related
I have two python files. Here is my ml.py file.
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn.linear_model import LinearRegression
import pickle
data = pd.read_csv("C:\\Users\\ryans\\Desktop\\insurance.csv")
le = LabelEncoder()
le.fit(data['sex'])
data['Sex'] = le.transform(data['sex'])
le.fit(data['smoker'])
data['Smoker'] = le.transform(data['smoker'])
le.fit(data['region'])
data['Region'] = le.transform(data['region'])
#independent and dependent columns
x = data[["age", "bmi", "children", "Sex", "Smoker", "Region"]]
y = data['charges']
#split in train and test
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.3, random_state=0)
#model training
linreg = LinearRegression()
linreg.fit(x_train, y_train)
#model testing
predictions = linreg.predict(x_test)
linreg.score(x_test,y_test)
#save the model
file = open("C:\\Users\\ryans\\Desktop\\expense_model.pkl", 'wb')
pickle.dump(linreg, file)
Here is my run.py file.
from flask import Flask, render_template, request
import pickle
app = Flask(__name__)
model = pickle.load(open('C:\\Users\\ryans\\Desktop\\expense_model.pkl','rb')) #read mode
#app.route("/")
def home():
return render_template('index.html')
#app.route("/predict", methods=['GET','POST'])
def predict():
if request.method == 'POST':
#access the data from form
## Age
age = int(request.form["age"])
bmi = int(request.form["bmi"])
children = int(request.form["children"])
Sex = int(request.form["Sex"])
Smoker = int(request.form["Smoker"])
Region = int(request.form["Region"])
#get prediction
input_cols = [[age, bmi, children, Sex, Smoker, Region]]
prediction = model.predict(input_cols)
output = round(prediction[0], 2)
return render_template("index.html", prediction_text='Your predicted annual Healthcare Expense is $ {}'.format(output))
if __name__ == "__main__":
app.run(debug=True)
Now, when I try to run run.py in my Anaconda Prompt, I get this error message.
(base) C:\Users\ryans\Desktop\run.py
Traceback (most recent call last):
File "C:\Users\ryans\Desktop\run.py", line 5, in <module>
model = pickle.load(open('C:\\Users\\ryans\\Desktop\\expense_model.pkl','rb')) #read mode
ModuleNotFoundError: No module named 'sklearn'
Sklearn is installed and it works fine when I run it in Spyder. I think, somehow, Flask is not finding the Sklearn correctly. Or, something isn't exposed. Or, rights are not set right. I don't know what's going on with Flask. This is my first time using it. Sklearn is working fine. I know that. Any thoughts on what's wrong here?
You haven't imported sklearn in run.py, so it can't unpickle the sklearn objects in your file.
I create a custom Pipeline in python. I have used the sklearn pipeline and it seems running successfully.
But When I save the model as a pickle file and want to load that saved pickle file in a different notebook it shows an error.
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import LabelEncoder
from sklearn import metrics
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
import eli5
from eli5.sklearn import PermutationImportance
from sklearn.pipeline import Pipeline, make_pipeline, FeatureUnion
from sklearn.preprocessing import FunctionTransformer
from sklearn.compose import ColumnTransformer
path = 'C:/Users/Desktop/'
df = pd.read_excel (path + "df.xlsx", sheet_name='df')
###################################################################################
# import the BaseEstimator
from sklearn.base import BaseEstimator
# define the class OutletTypeEncoder
# custom transformer must have methods fit and transform
class OutletTypeEncoder(BaseEstimator):
def __init__(self):
pass
#self.name = name
def fit(self, df, y=None):
return self
def transform(self, df):
# replace NaN
df[['pdf_tbl_pn_identifier', 'pdf_tbl_qty_identifier', 'pdf_header_present']] = df[['pdf_tbl_pn_identifier', 'pdf_tbl_qty_identifier', 'pdf_header_present']].fillna(value=-999)
df[['pdf_tbl_cnt']] = df[['pdf_tbl_cnt']].fillna(value=0)
# Replace gt 1 count as 0
df['pdf_tbl_cnt'] = np.where((df['pdf_tbl_cnt'] == '1'), 1, 0)
df['part_cnt'] = np.where((df['part_cnt'] == '1'), 1, 0)
# create numeric and categorica coulmns
obj_df= df[['pdf_tbl_pn_identifier','pdf_tbl_qty_identifier','pdf_header_present',
'pdf_body_pn_identifier','pdf_body_qty_identifier','pdf_model_rel_returned','pdf_model_ent_returned']]
num_df= df[['pdf_tbl_cnt', 'pdf_model_avg_relationship_score','pdf_model_avg_entity_score','part_cnt']]
# Labelencoding for categorica columns and then
obj_df=obj_df.apply(LabelEncoder().fit_transform)
df = pd.concat([obj_df, num_df], axis=1)
#df.reset_index(inplace=True, drop=True)
df.pdf_tbl_pn_identifier = df.pdf_tbl_pn_identifier.astype(str)
df.pdf_tbl_qty_identifier = df.pdf_tbl_qty_identifier.astype(str)
df.pdf_body_pn_identifier = df.pdf_body_pn_identifier.astype(str)
df.pdf_body_qty_identifier = df.pdf_body_qty_identifier.astype(str)
df.pdf_model_rel_returned = df.pdf_model_rel_returned.astype(str)
df.pdf_model_ent_returned = df.pdf_model_ent_returned.astype(str)
df.pdf_header_present = df.pdf_header_present.astype(str)
#df.matching = df.matching.astype(str)
#df['pdf_tbl_cnt'] = df['pdf_tbl_cnt'].apply(np.int64)
df.pdf_tbl_cnt = df.pdf_tbl_cnt.apply(np.int64)
return df
#################################################################################
feature_cols = df.drop(['matching'], axis=1)
X = feature_cols # Features
y = df.matching # Target variable
# split into train test sets
X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.25,random_state=0)
# Create Pipeline
logreg = LogisticRegression()
model_pipeline = Pipeline(steps=[('preprocess', OutletTypeEncoder()),
('logreg', LogisticRegression())
])
# fit the pipeline with the training data
model_pipeline.fit(X_train,y_train)
# Predict
y_pred=model_pipeline.predict(X_test)
Now I save the model as a pickle file and want to use that pickle file in another notebook.
But got an error that:
AttributeError: Can't get attribute 'OutletTypeEncoder' on <module 'main'>
# Save the Modle to file in the current working directory
Pkl_Filename = "C:\\Users\\SafayetKarim\\Desktop\\confidence_score\\results_updated\\pdf\\logisic_Model_pipeline.pkl"
with open(Pkl_Filename, 'wb') as file:
pickle.dump(model_pipeline, file)
# Load the Model back from file
with open('C:\\Users\\SafayetKarim\\Desktop\\confidence_score\\results_updated\\pdf\\logisic_Model_pipeline.pkl', 'rb') as file:
logisic_Model_pipeline = pickle.load(file)
logisic_Model_pipeline
Please help me out how to resolve the issue.
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-4-28376e81d621> in <module>
1 # Load the Model back from file
2 with open('C:\\Users\\SafayetKarim\\Desktop\\confidence_score\\results_updated\\pdf\\OutletTypeEncoder.pkl', 'rb') as file:
----> 3 OutletTypeEncoder = pickle.load(file)
4
5 OutletTypeEncoder
AttributeError: Can't get attribute 'OutletTypeEncoder' on <module '__main__'>
I am trying to deploy one ML model using flask in Heroku. Here are the files I have:
model.py, app.py, requirement.txt, procfile
Github link-: https://github.com/dee-walia20/clickbait_detector
Model.py code snippet
# Importing the libraries
import pandas as pd
import pickle
from sklearn.base import BaseEstimator, TransformerMixin
import nltk
import re
from nltk.stem import WordNetLemmatizer
from nltk.corpus import wordnet, stopwords
from sklearn.pipeline import make_pipeline
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
#Import dataset and assign Input & Output
data = pd.read_csv('clickbait_data.csv')
X=data.headline
y=data.clickbait
#Custom Transformer built from sklearn for Text Processing
class TextPreprocessor(BaseEstimator, TransformerMixin):
def __init__(self, lemmatizer=None, stopwords=None,token=True):
self.token = token
self.lemmatizer=lemmatizer
self.stopwords=stopwords
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
if self.token:
nltk.download('averaged_perceptron_tagger')
nltk.download('wordnet')
X=X.apply(lambda x: re.sub('\d+',"",x))
X=X.apply(lambda x: re.sub('\W'," ",x))
X=X.apply(lambda x: re.sub(" "," ",x))
X=X.apply(lambda x: x.lower())
self.lemmatizer=WordNetLemmatizer()
nltk.download("stopwords")
self.stopwords=stopwords.words('english')
X=X.apply(lambda x: " ".join([self.lemmatizer.lemmatize(word, self._get_wordnet_pos(word))
for word in x.split() if word not in self.stopwords]))
return X
def _get_wordnet_pos(self, word):
tag=nltk.pos_tag([word])[0][1][0].upper()
tag_dict={'J':wordnet.ADJ,
'N':wordnet.NOUN,
'V':wordnet.VERB,
'R':wordnet.ADV
}
return tag_dict.get(tag,wordnet.NOUN)
#Splitting Training and Test Set
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.20, random_state=10)
#Creating the pipeline for End to end ML model
tp=TextPreprocessor()
tf=TfidfVectorizer()
model=MultinomialNB()
pipeline=make_pipeline(tp,tf,model)
#Fitting model with trainig data
pipeline.fit(X_train, y_train)
# Saving model to disk
pickle.dump(pipeline, open('pipeline.pkl','wb'), protocol=-1)
# Loading model to check performance on Test set
if __name__ == "__main__":
saved_pipeline = pickle.load(open('pipeline.pkl','rb'))
print("Classitication Report\n",classification_report(y_test, saved_pipeline.predict(X_test)))
app.py
import pandas as pd
from flask import Flask, request, render_template
import pickle
from model import TextPreprocessor
app = Flask(__name__)
model = pickle.load(open('pipeline.pkl', 'rb'))
#app.route('/')
def home():
return render_template('index.html')
#app.route('/predict',methods=['GET','POST'])
def predict():
'''
For rendering results on HTML GUI
'''
final_features = pd.Series(data=str(request.form['headline']))
prediction = model.predict(final_features)
output = prediction[0]
return render_template('result.html', prediction=output)
if __name__ == "__main__":
app.run(debug=True)
Error message:
2020-05-02T11:25:36.304845+00:00 heroku[router]: at=error code=H12
desc="Request timeout" method=GET path="/favicon.ico"
host=clickbait-headline-detector.herokuapp.com
request_id=3aa2f839-109a-4c41-a121-3738e440a62f fwd="180.151.118.230"
dyno=web.1 connect=1ms service=30001ms status=503 bytes=0
protocol=https
I have read about 30ms timeout in Heroku for H12 error and possible solution to create background/worker taks. But I am not aware which task would be worker here and how to do that?
I have written this small machine learning code of a simple random forest regression in the class Model. After creating an object of this class I have printed the predictions and the accuracy score along with that I have written a code to schedule training every 30 days and testing every 7 days. But I'm facing an error
Code:
import schedule
import time
from sklearn.ensemble import RandomForestRegressor
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
import numpy as np
import pandas as pd
from main import data as df
class Model():
def __init__(self):
self.df = df
self.linear_reg = LinearRegression()
self.random_forest = RandomForestRegressor()
def split(self, test_size):
X = np.array(self.df[['age','experience','certificates']])
y = np.array(self.df['salary'])
self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(X, y, test_size = test_size, random_state = 42)
def fit(self):
self.model = self.random_forest.fit(self.X_train, self.y_train)
def predict(self):
self.result = self.random_forest.predict(self.X_test)
print(self.result)
print("Accuracy: ", self.model.score(self.X_test, self.y_test))
if __name__ == '__main__':
model_instance = Model()
model_instance.split(0.2)
schedule.every(30).days.at("05:00").do(model_instance.fit())
schedule.every(7).days.at("05:00").do(model_instance.predict())
while 1:
schedule.run_pending()
time.sleep(1)
On this line schedule.every(30).days.at("05:00").do(model_instance.fit()) I'm getting the following error: the first argument must be callable
I'm not familiar with the schedule package, but I guess the argument to do must be a callable. Which means you shouldn't actually call that function. Try this:
schedule.every(30).days.at("05:00").do(model_instance.fit)
schedule.every(7).days.at("05:00").do(model_instance.predict)
Note I removed the parentheses after fit and predict.
I figured it out. Created separate modules for training and testing and then imported the Model class and then created a function which will perform the scheduling.
Function for Training:
import schedule
import time
def job():
model_instance.split(0.2)
model_instance.fit()
print("Training Completed")
schedule.every().minute.at(":17").do(job)
while True:
schedule.run_pending()
time.sleep(1)
Function for testing:
import schedule
import time
def job():
model_instance.predict()
print(model_instance.result)
print("Accuracy: ", model_instance.model.score(model_instance.X_test, model_instance.y_test))
print("Testing Completed")
schedule.every().minute.at(":17").do(job)
while True:
schedule.run_pending()
time.sleep(1)
I want to predict Knoell data using ppar_ligand_model.
But some SMILES display this error message.
How can I handle it? Is this problem of SMILE's structure?
And What code should I add to exclude the material's SMILE that causes the error?
Here is soruce code and github link
run_pparg_ligand_model.py
import numpy
import pandas
import sys
from keras.models import Sequential
from keras.layers import Dense
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import StratifiedKFold
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.metrics import confusion_matrix
from rdkit.Chem import AllChem
from rdkit import DataStructs
from rdkit.Chem import Draw
from rdkit.Chem.Draw import SimilarityMaps
from rdkit import Chem, DataStructs
import seaborn as sns
import matplotlib as mpl
import matplotlib.pyplot as plt
from keras.models import load_model
def main():
"""Run the main function."""
outfile = open(sys.argv[1] + "_predictions.txt", 'w')
outfile.write("Chemical Name\tPrediction\n")
model = load_model("pparg_ligand_model.h5")
dataframe = pandas.read_csv(sys.argv[1], sep="\t")
mols = []
fps = []
for index, row in dataframe.iterrows():
# try :
# mol = Chem.MolFromSmiles(row['SMILES'])
# break
# except :
# continue
mol = Chem.MolFromSmiles(row['SMILES'])
fp = AllChem.GetMorganFingerprintAsBitVect(mol, 2)
mols.append(mol)
fps.append(fp)
np_fps = []
for fp in fps:
arr = numpy.zeros((1,))
DataStructs.ConvertToNumpyArray(fp, arr)
np_fps.append(arr)
np_fps_array = numpy.array(np_fps)
predictions = model.predict(np_fps_array, batch_size=5)
i = 0
for prediction in predictions:
y_prediction = ''
if(prediction < 0.50):
y_prediction = "ligand"
else:
y_prediction = "not_ligand"
outfile.write(dataframe['Chemical_Name'][i] + "\t" + y_prediction + "\n")
i += 1
outfile.close()
if __name__ == "__main__":
main()