Develop Reference

TF2 transform can't find an actuall existing frame

In a global planner node that I wrote, I have the following init code
#!/usr/bin/env python
import rospy
import copy
import tf2_ros
import time
import numpy as np
import math
import tf
from math import sqrt, pow
from geometry_msgs.msg import Vector3, Point
from std_msgs.msg import Int32MultiArray
from std_msgs.msg import Bool
from nav_msgs.msg import OccupancyGrid, Path
from geometry_msgs.msg import PoseStamped, PointStamped
from tf2_geometry_msgs import do_transform_point
from Queue import PriorityQueue
class GlobalPlanner():
def __init__(self):
print("init global planner")
self.tfBuffer = tf2_ros.Buffer()
self.listener = tf2_ros.TransformListener(self.tfBuffer)
self.drone_position_sub = rospy.Subscriber('uav/sensors/gps', PoseStamped, self.get_drone_position)
self.drone_position = []
self.drone_map_position = []
self.map_sub = rospy.Subscriber("/map", OccupancyGrid, self.get_map)
self.goal_sub = rospy.Subscriber("/cell_tower/position", Point, self.getTransformedGoal)
self.goal_position = []
self.goal = Point()
self.goal_map_position = []
self.occupancy_grid = OccupancyGrid()
self.map = []
self.p_path = Int32MultiArray()
self.position_pub = rospy.Publisher("/uav/input/position", Vector3, queue_size = 1)
#next_movement in
self.next_movement = Vector3
self.next_movement.z = 3
self.path_pub = rospy.Publisher('/uav/path', Int32MultiArray, queue_size=1)
self.width = rospy.get_param('global_planner_node/map_width')
self.height = rospy.get_param('global_planner_node/map_height')
#Check whether there is a path plan
self.have_plan = False
self.path = []
self.euc_distance_drone_goal = 100
self.twod_distance_drone_goal = []
self.map_distance_drone_goal = []
self.mainLoop()
And there is a call-back function call getTransformed goal, which will take the goal position in the "cell_tower" frame to the "world" frame. Which looks like this
def getTransformedGoal(self, msg):
self.goal = msg
try:
#Lookup the tower to world transform
transform = self.tfBuffer.lookup_transform('cell_tower', 'world', rospy.Time())
#transform = self.tfBuffer.lookup_transform('world','cell-tower' rospy.Time())
#Convert the goal to a PointStamped
goal_pointStamped = PointStamped()
goal_pointStamped.point.x = self.goal.x
goal_pointStamped.point.y = self.goal.y
goal_pointStamped.point.z = self.goal.z
#Use the do_transform_point function to convert the point using the transform
new_point = do_transform_point(goal_pointStamped, transform)
#Convert the point back into a vector message containing integers
transform_point = [new_point.point.x, new_point.point.y]
#Publish the vector
self.goal_position = transform_point
except (tf2_ros.LookupException, tf2_ros.ConnectivityException, tf2_ros.ExtrapolationException) as e:
print(e)
print('global_planner tf2 exception, continuing')
The error message said that
"cell_tower" passed to lookupTransform argument target_frame does not exist.
I check the RQT plot for both active and all, which shows that when active, the topic /tf is not being subscribe by the node global planner. Check the following image, which is for active
enter image description here
and this image is for all the node (include non-active)
enter image description here
But I have actually set up the listner, I have another node call local planner that use the same strategy and it works for that node, but not for the global planner
I'm not sure why this is.

Try adding a timeout to your lookup_transform() function call, as your transformation may not be available when you need it:
transform = self.tfBuffer.lookup_transform('cell_tower', 'world',rospy.Time.now(), rospy.Duration(1.0))

Jupyter Notebook is stuck on "Loading..."

So I am trying to run this pymongo app through jupyter notebook and all it does is give me a "Loading..." output and then the kernel goes idle... when I just print the "df", it gives me the data so I know that it is working data-wise. I have tried restarting the kernel; deleting the output, restarting the kernel, and reinputting the input; Restarting and clearing output - and all do not seem to have any effect. I am thinking it may have something to do with my inputted data but I am not sure what exactly...
This is the .ipynb:
from jupyter_plotly_dash import JupyterDash
import dash
import dash_leaflet as dl
import dash_core_components as dcc
import dash_html_components as html
import plotly.express as px
import dash_table
from dash.dependencies import Input, Output
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from pymongo import MongoClient
#### FIX ME #####
# change animal_shelter and AnimalShelter to match your CRUD Python module file name and class name
from AAC import AnimalShelter
###########################
# Data Manipulation / Model
###########################
# FIX ME update with your username and password and CRUD Python module name
username = "aacuser"
password = "monogoadmin"
shelter = AnimalShelter(username, password)
# class read method must support return of cursor object and accept projection json input
df = pd.DataFrame.from_records(shelter.read({}))
#########################
# Dashboard Layout / View
#########################
app = JupyterDash('Dash DataTable Only')
app.layout = html.Div([
html.Div(id='hidden-div', style={'display':'none'}),
html.Center(html.B(html.H1('SNHU CS-340 Dashboard'))),
html.Hr(),
dash_table.DataTable(
id='datatable-interactivity',
columns=[
{"name": i, "id": i, "deletable": False, "selectable": True} for i in df.columns
],
data=df.to_dict('records'),
#FIXME: Set up the features for your interactive data table to make it user-friendly for your client
editable=False,
filter_action="native",
sort_action="native",
sort_mode="multi",
column_selectable=False,
row_selectable=False,
row_deletable=False,
selected_columns=[],
selected_rows=[],
page_action="native",
page_current= 0,
page_size= 10,
),
html.Br(),
html.Hr(),
])
app
This is the .py file:
import pymongo
from pymongo import MongoClient
from bson.objectid import ObjectId
class AnimalShelter(object):
"""CRUD operations for Animal collection in Mongodatabase"""
#Initializes MongoClient
def __init__(self, username, password):
self.client = MongoClient('mongodb://127.0.0.1:38574', username='aacuser', password='mongoadmin', authSource='AAC', authMechanism='SCRAM-SHA-1')
self.database = self.client['AAC']
#Implement create method
def create(self, data):
if data is not None:
return self.database.animals.insert_one(data)
else:
raise Exception("Nothing to save, because data parameter is empty")
#Implement read method
def read(self, data):
if data is not None:
return self.database.animals.find(data)
else:
raise Exception("Nothing to read, because data parameter is empty")
#Implement update method
def update(self, data):
if find is not None:
return self.database.animals.update_one(data)
else:
raise Exception("Nothing to update, because data parameter is empty")
#Implement delete method
def delete(self, data):
if data is not None:
return self.database.animals.delete_one(data)
else:
raise Exception("Nothing to delete, because data parameter is empty")
Any help is greatly appreciated!
EDIT:
Here is the requested screen shot
I also wanted to note that I cleared cache and cookies and this had no effect.

making a function that run after certain condition in python flask

I want to build a Line chatbot that when a POST requests is received and will parse the receive POST requests information to the chatbot users.
import os
import sys
from argparse import ArgumentParser
import json
from flask import Flask, request, abort,Response,jsonify
import requests
from linebot import (
LineBotApi, WebhookHandler
)
from linebot.exceptions import (
InvalidSignatureError, LineBotApiError
)
from linebot.models import (
MessageEvent, TextMessage, TextSendMessage,SourceUser
)
app = Flask(__name__)
data=''
#app.route('/data', methods=['POST'])
def get_data():
"""devicename,exceptionType,temperature,mask,stranger"""
device_name = request.form.get('device_name')
exceptionType = request.form.get('exceptionType')
temperature = request.form.get('temperature')
mask = request.form.get('mask')
stranger = request.form.get('stranger')
global data
data=f"device_name = {device_name} exceptionType = {exceptionType} temperature = {temperature} mask = {mask} stranger = {stranger}"
return_status = {"success":"true"}
return jsonify(return_status)
while len(data) > 1:
try:
line_bot_api.broadcast(
messages=TextSendMessage(text=str(data))
)
data=''
except LineBotApiError as e:
raise e
if __name__ == "__main__":
arg_parser = ArgumentParser(
usage='Usage: python ' + __file__ + ' [--port <port>] [--help]'
)
arg_parser.add_argument('-p', '--port', default=8000, help='port')
arg_parser.add_argument('-d', '--debug', default=False, help='debug')
options = arg_parser.parse_args()
app.run(debug=options.debug, port=options.port)
here I have two parts, first is the POST requests receiver. and second is the part that checks the condition if data is present. But when I send the POST requests, the line_bot_api.broadcast does not send the message to the chatbot user. I only saw examples of flask code that has a #app.route() so I was wondering if something like this works? or is there a better way to implement the condition (instead of checking data value, is it better to check if function get_data() ran or requests has been sent to our /data route)

The while loop stops immediately because the condition fails. One way to get around this and do what you want is by using threads.
import os
import sys
from argparse import ArgumentParser
import json
import threading
from flask import Flask, request, abort,Response,jsonify
import requests
from linebot import (
LineBotApi, WebhookHandler
)
from linebot.exceptions import (
InvalidSignatureError, LineBotApiError
)
from linebot.models import (
MessageEvent, TextMessage, TextSendMessage,SourceUser
)
app = Flask(__name__)
data=''
#app.route('/data', methods=['POST'])
def get_data():
"""devicename,exceptionType,temperature,mask,stranger"""
device_name = request.form.get('device_name')
exceptionType = request.form.get('exceptionType')
temperature = request.form.get('temperature')
mask = request.form.get('mask')
stranger = request.form.get('stranger')
global data
data=f"device_name = {device_name} exceptionType = {exceptionType} temperature = {temperature} mask = {mask} stranger = {stranger}"
return_status = {"success":"true"}
return jsonify(return_status)
def broadcast():
while True:
if len(data) < 1 : continue
try:
line_bot_api.broadcast(
messages=TextSendMessage(text=str(data))
)
data=''
except LineBotApiError as e:
raise e
if __name__ == "__main__":
arg_parser = ArgumentParser(
usage='Usage: python ' + __file__ + ' [--port <port>] [--help]'
)
arg_parser.add_argument('-p', '--port', default=8000, help='port')
arg_parser.add_argument('-d', '--debug', default=False, help='debug')
options = arg_parser.parse_args()
broadcast_thread = threading.Thread(target=broadcast)
broadcast_thread.start()
app.run(debug=options.debug, port=options.port)
This is definitely not the best way. But it works.

Passing a pandas dataframe to FastAPI for NLP ML

I am trying to, for the first time, deploy an NLP ML model. To do this it was suggested that I use FastAPI and uvicorn. I have had some success in getting FastAPI to respond; however, I have not been able to successfully pass the dataframe and have it process it. I've tried using dictionaries and even attempted to convert the passed json to a dataframe.
With data_dict = data.dict() I get:
ValueError: Iterable over raw text documents expected, string object received.
With data_dict = pd.DataFrame(data.dict()) I get:
ValueError: If using all scalar values, you must pass an index
I believe I understand the problem, my Data class is expecting a string which this is not; however, I have not been able to determine how to set and / or pass the expected data so that fit_transform() will work. Ultimately I will have a prediction returned based on the submitted messages value. Bonus if I can pass a dataframe of 1 or more rows and have predictions made and returned for each of the rows. The response will include the id, project, and the prediction so that we are in future able to leverage this response to post the prediction back to the original (requesting) system.
test_connection.py
#%%
import requests
import pandas as pd
import json
import os
from pprint import pprint
url = 'http://127.0.0.1:8000/predict'
print(os.getcwd())
#%%
df = pd.DataFrame(
{
'id': ['ab410483801c38', 'cd34148639180'],
'project': ['project1', 'project2'],
'messages': ['This is message 1', 'This is message 2']
}
)
to_predict_dict = df.iloc[0].to_dict()
#%%
r = requests.post(url, json=to_predict_dict)
main.py
#!/usr/bin/env python
# coding: utf-8
import pickle
import pandas as pd
import numpy as np
from pydantic import BaseModel
from sklearn.feature_extraction.text import TfidfVectorizer
# Server
import uvicorn
from fastapi import FastAPI
# Model
import xgboost as xgb
app = FastAPI()
clf = pickle.load(open('data/xgbmodel.pickle', 'rb'))
class Data(BaseModel):
# id: str
project: str
messages: str
#app.get("/ping")
async def test():
return {"ping": "pong"}
#app.post("/predict")
async def predict(data: Data):
# data_dict = data.dict()
data_dict = pd.DataFrame(data.dict())
tfidf_vect = TfidfVectorizer(stop_words="english", analyzer='word', token_pattern=r'\w{1,}')
tfidf_vect.fit_transform(data_dict['messages'])
# to_predict = tfidf_vect.transform(data_dict['messages'])
# prediction = clf.predict(to_predict)
return {"response": "Success"}

Probably not the most elegant solution but I've made progress using the following:
def predict(data: Data):
data_dict = pd.DataFrame(
{
'id': [data.id],
'project': [data.project],
'messages': [data.messages]
}
)

Frist, encode your dataFrame df to JSON record-oriented:
r = requests.post(url, json=df.to_json(orient='records')).
Then, decode your data inside the /predict/ endpoint with:
df = pd.DataFrame(jsonable_encoder(data))
Remember to import the module from fastapi.encoders import jsonable_encoder.

A new library called pandera now supports direct passage of DataFrames without conversion via FastAPI. The docs are bit basic as of posting this, but may be worth reading: https://pandera.readthedocs.io/en/latest/fastapi.html#fastapi-integration.

I was able to address the issue by simply converting data.messages into a list. I also had to make some unrelated changes, I had failed to pickle my vectorizer (string tokenizer).
import pickle
import pandas as pd
import numpy as np
import json
import time
from pydantic import BaseModel
from sklearn.feature_extraction.text import TfidfVectorizer
# Server / endpoint
import uvicorn
from fastapi import FastAPI
# Model
import xgboost as xgb
app = FastAPI(debug=True)
clf = pickle.load(open('data/xgbmodel.pickle', 'rb'))
vect = pickle.load(open('data/tfidfvect.pickle', 'rb'))
class Data(BaseModel):
id: str = None
project: str
messages: str
#app.get("/ping")
async def ping():
return {"ping": "pong"}
#app.post("/predict/")
def predict(data: Data):
start = time.time()
data_l = [data.messages] # make messages iterable.
to_predict = vect.transform(data_l)
prediction = clf.predict(to_predict)
exec_time = round((time.time() - start), 3)
return {
"id": data.id,
"project": data.project,
"prediction": prediction[0],
"execution_time": exec_time
}
if __name__ == "__main__":
uvicorn.run(app, host="127.0.0.1", port=8000)

How to export account portfolio information at Interactive Brokers

I would like to use a dataframe to store my portfolio information and update every minute.
But the outcome is empty from the below code, am I missing something?
from ibapi.client import EClient
from ibapi.wrapper import EWrapper
from ibapi.contract import Contract
from ibapi.ticktype import TickTypeEnum
import pandas as pd
import time
class IBapi(EWrapper, EClient):
def __init__(self):
EClient.__init__(self, self)
self.all_positions = pd.DataFrame([], columns=['ConID', 'Symbol', 'Quantity', 'Average Cost', 'MarketPrice', 'marketValue', 'unrealizedONL', 'realizedPNL'])
def updatePortfolio(self, contract: Contract, position: float, marketPrice: float, marketValue: float,averageCost: float, unrealizedPNL: float, realizedPNL:float, accountName:str):
super().updatePortfolio(contract, position, marketPrice, marketValue, averageCost, unrealizedPNL, realizedPNL, accountName)
index = str(contract.conId)
self.all_positions.loc[index] = contract.conId, contract.symbol, position, averageCost, marketPrice, marketValue, unrealizedPNL, realizedPNL
def main():
app = IBapi()
app.connect('127.0.0.1', 7497, 0)
app.reqAccountUpdates(True, "XXXXXXXX")
current_positions = app.reqAccountUpdates(True, "XXXXXXX")
app.run()
print(current_positions.to_string())
app.disconnect()
if __name__ == "__main__":
main()

reqAcccountUpdates is an asynchronous function call - it sends an outgoing message but does not wait for a response. (Since the message initiates a subscription to a data stream, there isn't a single response returned but instead a series of responses).
So current_positions in:
current_positions = app.reqAccountUpdates(True, "XXXXXXX")
will always be None. Instead the responses are stored in app.all_positions by the overridden updatePortfolio() function.
Also, the run() loop is an infinite loop so the lines after that won't execute. Most commonly this type of architecture would be handled by python futures/the asyncio module (as in the ib_insync library), or with an additional thread for the run loop.