This code is from here
I have the following code for a telegram bot which i am building:
import pandas as pd
from pandas import datetime
from pandas import DataFrame as df
import matplotlib
from pandas_datareader import data as web
import matplotlib.pyplot as plt
import datetime
import requests
from bottle import (
run, post, response, request as bottle_request
)
BOT_URL = 'https://api.telegram.org/bot128secretns/'
def get_chat_id(data):
"""
Method to extract chat id from telegram request.
"""
chat_id = data['message']['chat']['id']
return chat_id
def get_message(data):
"""
Method to extract message id from telegram request.
"""
message_text = data['message']['text']
return message_text
def send_message(prepared_data):
"""
Prepared data should be json which includes at least `chat_id` and `text`
"""
message_url = BOT_URL + 'sendMessage'
requests.post(message_url, json=prepared_data)
def get_ticker(text):
stock = f'^GSPC'
start = datetime.date(2000,1,1)
end = datetime.date.today()
data = web.DataReader(stock, 'yahoo',start, end)
plot = data.plot(y='Open')
return plot
def prepare_data_for_answer(data):
answer = get_ticker(get_message(data))
json_data = {
"chat_id": get_chat_id(data),
"text": answer,
}
return json_data
#post('/')
def main():
data = bottle_request.json
answer_data = prepare_data_for_answer(data)
send_message(answer_data) # <--- function for sending answer
return response # status 200 OK by default
if __name__ == '__main__':
run(host='localhost', port=8080, debug=True)
When i run this code i am getting the following error:
TypeError: Object of type AxesSubplot is not JSON serializable
What this code is suppose to do is take ticker symbols from telegram app and return its chart back.
I know this is because json does not handle images.
What can i do to resolve it?
Sorry, I'm a bit late to the party. Here is a possible solution below, though I didn't test it. Hope it works or at least gives you a way to go about solving the issue :)
import datetime
from io import BytesIO
import requests
from pandas_datareader import data as web
from bottle import (
run, post, response, request as bottle_request
)
BOT_URL = 'https://api.telegram.org/bot128secretns/'
def get_chat_id(data):
"""
Method to extract chat id from telegram request.
"""
chat_id = data['message']['chat']['id']
return chat_id
def get_message(data):
"""
Method to extract message id from telegram request.
"""
message_text = data['message']['text']
return message_text
def send_photo(prepared_data):
"""
Prepared data should be json which includes at least `chat_id` and `plot_file`
"""
data = {'chat_id': prepared_data['chat_id']}
files = {'photo': prepared_data['plot_file']}
requests.post(BOT_URL + 'sendPhoto', json=data, files=files)
def get_ticker(text):
stock = f'^GSPC'
start = datetime.date(2000,1,1)
end = datetime.date.today()
data = web.DataReader(stock, 'yahoo',start, end)
plot = data.plot(y='Open')
return plot
def prepare_data_for_answer(data):
plot = get_ticker(get_message(data))
# Write the plot Figure to a file-like bytes object:
plot_file = BytesIO()
fig = plot.get_figure()
fig.savefig(plot_file, format='png')
plot_file.seek(0)
prepared_data = {
"chat_id": get_chat_id(data),
"plot_file": plot_file,
}
return prepared_data
#post('/')
def main():
data = bottle_request.json
answer_data = prepare_data_for_answer(data)
send_photo(answer_data) # <--- function for sending answer
return response # status 200 OK by default
if __name__ == '__main__':
run(host='localhost', port=8080, debug=True)
The idea is not to send a message using the sendMessage Telegram API endpoint, but to send a photo file by using the sendPhoto endpoint. Here, we use savefig call in the prepare_data_for_answer function body to convert AxesSubplot instance, that we get as a return value from the get_ticker function, to a file-like BytesIO object, which we then send as a photo to Telegram using send_photo function (previously named as send_message).
You may use bob-telegram-tools
from bob_telegram_tools.bot
import TelegramBot
import matplotlib.pyplot as plt
token = '<your_token>'
user_id = int('<your_chat_id>')
bot = TelegramBot(token, user_id)
plt.plot([1, 2, 3, 4])
plt.ylabel('some numbers')
bot.send_plot(plt)
# This method delete the generetad image
bot.clean_tmp_dir()
You cannot send a matplotlib figure directly. You will need to convert it to bytes and then send it as a multipart message.
data.plot will return a matplotlib.axes.Axes object. You can save convert the figure to bytes like this
import StringIO
img = StringIO.StringIO()
plot.fig.savefig(img, format='png')
img.seek(0)
yukuku/telebot has some good code on how to send the image as a message. Check this line here.
Related
I tried everything, why my 'POST request' from 'send' function isn't considered in the 'checkview' function ?
def send(request):
room = request.POST['room_name']
#YAHOO API
import yfinance as yf
aapl = yf.Ticker(room)
ainfo = aapl.history(period='1y')
#Plot the Chart
import plotly.graph_objects as go
fig = go.Figure(data=go.Scatter(x=ainfo.index,y=ainfo.Close, mode='lines'))
#x = fig.show()
#DB inject plot
rx = str(randint(9999,99999))
fig.write_image("/Users/Xlibidish/Desktop/Django/static/"+room+rx+".png")
#DB Inject from send
plot_id = str(room+rx)
new_message = Message.objects.create(room=plot_id)
new_message.save()
return HttpResponse('All Good in send')
def checkview(self):
send(self)
chart_id = Message.objects.get(room=plot_id)
return JsonResponse(chart_id.room, safe=False)
Here is the error:
django.utils.datastructures.MultiValueDictKeyError: 'room_name'
I ultimately need to transfer through AJAX the plot_id variable.
Im trying to return a matplotlib.figure.Figure in FastAPI.
If I save it like an image it works (code here):
#router.get("/graph/{id_file}", name="Return the graph obtained")
async def create_graph(id_file: str):
data = HAR.createGraph(id_file)
graph = HAR.scatterplot(data['dateTimes'], data['label'], "Time", "Activity")
graph.savefig('saved_figure.jpg')
return FileResponse('saved_figure.jpg')
Where graph is my Figure.
But I would like to show it without saving in mi computer.
savefig can accept binary file-like object. It can be used to achieve what you want.
The code could be:
from io import BytesIO
from starlette.responses import StreamingResponse
...
#router.get("/graph/{id_file}", name="Return the graph obtained")
async def create_graph(id_file: str):
data = HAR.createGraph(id_file)
graph = HAR.scatterplot(data['dateTimes'], data['label'], "Time", "Activity")
# create a buffer to store image data
buf = BytesIO()
graph.savefig(buf, format="png")
buf.seek(0)
return StreamingResponse(buf, media_type="image/png")
I am trying to, for the first time, deploy an NLP ML model. To do this it was suggested that I use FastAPI and uvicorn. I have had some success in getting FastAPI to respond; however, I have not been able to successfully pass the dataframe and have it process it. I've tried using dictionaries and even attempted to convert the passed json to a dataframe.
With data_dict = data.dict() I get:
ValueError: Iterable over raw text documents expected, string object received.
With data_dict = pd.DataFrame(data.dict()) I get:
ValueError: If using all scalar values, you must pass an index
I believe I understand the problem, my Data class is expecting a string which this is not; however, I have not been able to determine how to set and / or pass the expected data so that fit_transform() will work. Ultimately I will have a prediction returned based on the submitted messages value. Bonus if I can pass a dataframe of 1 or more rows and have predictions made and returned for each of the rows. The response will include the id, project, and the prediction so that we are in future able to leverage this response to post the prediction back to the original (requesting) system.
test_connection.py
#%%
import requests
import pandas as pd
import json
import os
from pprint import pprint
url = 'http://127.0.0.1:8000/predict'
print(os.getcwd())
#%%
df = pd.DataFrame(
{
'id': ['ab410483801c38', 'cd34148639180'],
'project': ['project1', 'project2'],
'messages': ['This is message 1', 'This is message 2']
}
)
to_predict_dict = df.iloc[0].to_dict()
#%%
r = requests.post(url, json=to_predict_dict)
main.py
#!/usr/bin/env python
# coding: utf-8
import pickle
import pandas as pd
import numpy as np
from pydantic import BaseModel
from sklearn.feature_extraction.text import TfidfVectorizer
# Server
import uvicorn
from fastapi import FastAPI
# Model
import xgboost as xgb
app = FastAPI()
clf = pickle.load(open('data/xgbmodel.pickle', 'rb'))
class Data(BaseModel):
# id: str
project: str
messages: str
#app.get("/ping")
async def test():
return {"ping": "pong"}
#app.post("/predict")
async def predict(data: Data):
# data_dict = data.dict()
data_dict = pd.DataFrame(data.dict())
tfidf_vect = TfidfVectorizer(stop_words="english", analyzer='word', token_pattern=r'\w{1,}')
tfidf_vect.fit_transform(data_dict['messages'])
# to_predict = tfidf_vect.transform(data_dict['messages'])
# prediction = clf.predict(to_predict)
return {"response": "Success"}
Probably not the most elegant solution but I've made progress using the following:
def predict(data: Data):
data_dict = pd.DataFrame(
{
'id': [data.id],
'project': [data.project],
'messages': [data.messages]
}
)
Frist, encode your dataFrame df to JSON record-oriented:
r = requests.post(url, json=df.to_json(orient='records')).
Then, decode your data inside the /predict/ endpoint with:
df = pd.DataFrame(jsonable_encoder(data))
Remember to import the module from fastapi.encoders import jsonable_encoder.
A new library called pandera now supports direct passage of DataFrames without conversion via FastAPI. The docs are bit basic as of posting this, but may be worth reading: https://pandera.readthedocs.io/en/latest/fastapi.html#fastapi-integration.
I was able to address the issue by simply converting data.messages into a list. I also had to make some unrelated changes, I had failed to pickle my vectorizer (string tokenizer).
import pickle
import pandas as pd
import numpy as np
import json
import time
from pydantic import BaseModel
from sklearn.feature_extraction.text import TfidfVectorizer
# Server / endpoint
import uvicorn
from fastapi import FastAPI
# Model
import xgboost as xgb
app = FastAPI(debug=True)
clf = pickle.load(open('data/xgbmodel.pickle', 'rb'))
vect = pickle.load(open('data/tfidfvect.pickle', 'rb'))
class Data(BaseModel):
id: str = None
project: str
messages: str
#app.get("/ping")
async def ping():
return {"ping": "pong"}
#app.post("/predict/")
def predict(data: Data):
start = time.time()
data_l = [data.messages] # make messages iterable.
to_predict = vect.transform(data_l)
prediction = clf.predict(to_predict)
exec_time = round((time.time() - start), 3)
return {
"id": data.id,
"project": data.project,
"prediction": prediction[0],
"execution_time": exec_time
}
if __name__ == "__main__":
uvicorn.run(app, host="127.0.0.1", port=8000)
I'm successfully trained my own dataset using Keras yolov3 Github project link
and I've got good predictions:
I would like to deploy this model on the web using flask to make it work with a stream or with IP cameras.
I saw many tutorials explains how to do that but, in reality, I did not find what I am looking for.
How can I get started?
You can use flask-restful to design a simple rest API.
You can use opencv VideoCapture to grab the video stream and get frames.
import numpy as np
import cv2
# Open a sample video available in sample-videos
vcap = cv2.VideoCapture('URL')
The client will take an image/ frame, encode it using base64, add other details like height, width, and make a request.
import numpy as np
import base64
import zlib
import requests
import time
t1 = time.time()
for _ in range(1000): # 1000 continuous request
frame = np.random.randint(0,256, (416,416,3), dtype=np.uint8) # dummy rgb image
# replace frame with your image
# compress
data = frame # zlib.compress(frame)
data = base64.b64encode(data)
data_send = data
#data2 = base64.b64decode(data)
#data2 = zlib.decompress(data2)
#fdata = np.frombuffer(data2, dtype=np.uint8)
r = requests.post("http://127.0.0.1:5000/predict", json={'imgb64' : data_send.decode(), 'w': 416, 'h': 416})
# make a post request
# print the response here
t2 = time.time()
print(t2-t1)
Your server will load the darknet model, and when it receives a post request it will simply return the model output.
from flask import Flask, request
from flask_restful import Resource, Api, reqparse
import json
import numpy as np
import base64
# compression
import zlib
# load keras model
# load_model('model.h5')
app = Flask(__name__)
api = Api(app)
parser = reqparse.RequestParser()
parser.add_argument('imgb64', location='json', help = 'type error')
parser.add_argument('w', type = int, location='json', help = 'type error')
parser.add_argument('h', type = int, location='json', help = 'type error')
class Predict(Resource):
def post(self):
request.get_json(force=True)
data = parser.parse_args()
if data['imgb64'] == "":
return {
'data':'',
'message':'No file found',
'status':'error'
}
img = data['imgb64']
w = data['w']
h = data['h']
data2 = img.encode()
data2 = base64.b64decode(data2)
#data2 = zlib.decompress(data2)
fdata = np.frombuffer(data2, dtype=np.uint8).reshape(w, h, -1)
# do model inference here
if img:
return json.dumps({
'mean': np.mean(fdata),
'channel': fdata.shape[-1],
'message':'darknet processed',
'status':'success'
})
return {
'data':'',
'message':'Something when wrong',
'status':'error'
}
api.add_resource(Predict,'/predict')
if __name__ == '__main__':
app.run(debug=True, host = '0.0.0.0', port = 5000, threaded=True)
In the # do model inference here part, just use your detect/predict function.
If you want to use native darknet, https://github.com/zabir-nabil/tf-model-server4-yolov3
If you want to use gRPC instead of REST, https://github.com/zabir-nabil/simple-gRPC
I am trying to run python with html using flask locally, i try to render a template within a class object, it compiles but when i try to access it on http://localhost:5000 it gives me 404. Could anyone tell what I'm doing wrong here?
I am trying to display values from a json format using chart.js library.
from flask import Flask
from flask import render_template
import os.path
import json
app = Flask(__name__)
#import the json file
_player_json_file = os.path.join(os.path.dirname(__file__), 'players.json')
def read_players(player_file=None):
if player_file is None:
player_file = _player_json_file
try:
data = json.loads(open(player_file).read())
except IOError:
return {}
#make player dictionary
players = {}
#get player ID from json file = data.
for playerID in data:
players[playerID] = Player_data(data[playerID])
return players
class Players_Data (object):
#app.route("/")
def __init__(self, data):
"""
Assign all the values from the json file to new variables
the values are birth date age weight...
"""
self.player_ID = data['gsis_id']
self.gsis_name = data.get('gsis_name', '')
self.player_fullname = data.get('full_name', '')
self.player_first_name = data.get('first_name', '')
self.player_last_name = data.get('last_name', '')
self.player_weight = data.get('weight','')
self.player_height = data.get('height' , '')
self.player_birth = data.get('birthdate', '')
self.player_pro_years = data.get('years_pro', '')
self.player_team = data.get('data', '')
values = [player_ID,gsis_name,player_fullname,player_first_name,player_last_name,player_weight]
return render_template('chart.html', data=data, values=values)
if __name__ == "__main__":
app.run(host='localhost')
I don't think template rendering is the issue here. You can just return a string from your view as an example.
You get a 404 because that's not how you do classy views in flask. You can have a look here: http://flask.pocoo.org/docs/0.12/views/
But essentially you have to extend from flask.views.View and then manually attach your route to app instance, so it looks something like this:
from flask import Flask
from flask.views import View
app = Flask(__name__)
class Players_Data(View):
def dispatch_request(self):
return 'LOL'
app.add_url_rule('/', view_func=Players_Data.as_view('show_users'))
if __name__ == "__main__":
app.run(host='localhost')