I have a table stored in a database in MySQL.
I fetched the results using MySQL connector and copied it to DataFrame. There are no problems till that.
Now as I'm taking the results from MySQL, they are in the form of strings, so I've converted the strings to int for the values of CONFIRMED_CASES, and leave the STATE_NAME as str.
Now I want to plot it in a bar graph, with Numeric data as CONFIRMED_CASES and state names as STATE_NAME but it shows me this error:
Traceback (most recent call last):
File "c:\Users\intel\Desktop\Covid Tracker\get_sql_data.py", line 66, in <module>
fd.get_graph()
File "c:\Users\intel\Desktop\Covid Tracker\get_sql_data.py", line 59, in get_graph
ax = df.plot(y='STATE_NAME', x='CONFIRMED_CASES',
...
raise TypeError("no numeric data to plot")
TypeError: no numeric data to plot
Here's my code:
from operator import index
from sqlalchemy import create_engine
import mysql.connector
import pandas as pd
import matplotlib.pyplot as plt
mydb = mysql.connector.connect(
host="localhost",
user="abhay",
password="1234",
database="covid_db"
)
mycursor = mydb.cursor()
class fetch_data():
def __init__(self):
...
def get_data(self, cmd):
...
def get_graph(self):
mydb = mysql.connector.connect(
host="localhost",
user="abhay",
password="1234",
database="covid_db"
)
mycursor = mydb.cursor(buffered=True)
mycursor.execute(
"select CONFIRMED_CASES from india;")
query = mycursor.fetchall()
# the data was like 10,000 so I removed the ',' and converted it to int
query = [int(x[0].replace(',', '')) for x in query]
print(query)
query2 = mycursor.execute(
"select STATE_NAME from india;")
query2 = mycursor.fetchall()
# this is the query for state name and i kept it as str only...
query2 = [x[0] for x in query2]
print(query2)
df = pd.DataFrame({
'CONFIRMED_CASES': [query],
'STATE_NAME': [query2],
})
# it gives me the error here...
ax = df.plot(y='STATE_NAME', x='CONFIRMED_CASES',
kind='bar')
ax.ticklabel_format(style='plain', axis='y')
plt.show()
fd = fetch_data()
fd.get_graph()
I don't know why there is no numeric value. I set it to int, but still...
Your dataframe is defined using lists of a list. query and query2 are lists already. Define your dataframe as:
df = pd.DataFrame({
'CONFIRMED_CASES': query, # no brackets
'STATE_NAME': query2, # no brackets
})
Use below code for converting data to integer before plotting graph.
df['CONFIRMED_CASES'] = df['CONFIRMED_CASES'].astype('int64')
Heyy guys,
It's solved.
I used query.values and it worked...
I am trying to read the latest row from a table and plot the graph using animate and matplotlib in python. The table gets updated every 1 second with a new value. I need to simulate the graph by live plotting the values.
However, when I use animate function with an interval of 1 second, I get the same value for every interval fetch.
I am adding the code for your reference. Please let me know what I am missing. The same code is working good when I use a flat file instead of MySql table.
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from matplotlib import style
import mysql.connector
import pandas as pd
style.use('fivethirtyeight')
mydb = mysql.connector.connect(
host="xxxxxxxxxx",
user="xxxxx",
passwd="xxxxxxx",
database="sakila"
)
fig = plt.figure(figsize=(8,5))
ax = plt.subplot2grid((1,1), (0,0))
plt.ion()
cursor = mydb.cursor()
def animate(i):
df = pd.read_sql("SELECT * FROM DATA_TEST ORDER BY ID DESC LIMIT 1", mydb)
y = df["VALUE"]
x = df["ID"]
xs = []
ys = []
xs.append(x)
ys.append(float(y)*100)
ax.clear()
ax.plot(xs,ys)
ani = animation.FuncAnimation(fig, animate, interval=1000)
plt.show()
## TESTING CURSOR IN FOR LOOP
import mysql.connector
import pandas as pd
import time
mydb = mysql.connector.connect(
host="xxxxxxx",
user="xxxx",
passwd="xxxxxx",
database="xxxxx"
)
for i in range(10):
cursor = mydb.cursor()
cursor.execute("SELECT * FROM DATA_TEST ORDER BY ID DESC LIMIT 1")
result = cursor.fetchall()
print("Total rows are: ", len(result))
for j in result:
print(j[0])
print(j[1])
time.sleep(10)
cursor.close()
So I've been through stack overflow and have used solutions from other posts.
But no matter what I try my ticks always remain as just ints and don't adopt the datetime format i yearn for. Any help would be appreciated!
from datetime import datetime, time
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import ticker
import mpl_finance
import MySQLdb
def isolated_candle_graph():
#Access my database
conn = MySQLdb.connect(host='localhost',
user='root',
passwd='*******',#
db='********') # falsified information
cursor = conn.cursor()
sql = "SELECT * FROM ddr"
cursor.execute(sql)
result = cursor.fetchall()
df = pd.DataFrame(list(result), columns=["Date", "Open", "High","Low", "Close", "Adj_Close", "Volume"])
fig, ax = plt.subplots()
#the plot
mpl_finance.candlestick2_ohlc(ax, df.Open, df.High, df.Low, df.Close,
width=0.6, colorup='r', colordown='c', alpha=1)
xdate = df.index
def mydate(x, pos):
try:
return xdate[int(x)]
except IndexError:
return ''
ax.xaxis.set_major_formatter(ticker.FuncFormatter(mydate))
plt.show()
isolated_candle_graph()
Thanks!
This is what I've been following.
matplotlib.finance.candlestick_ohlc plot intraday 1min bar data with time breaks and proper xticklabels at every hour
This is the script that I wrote so far. The first blocker that I find is that I am not being able to install MySQLdb package - Maybe I could use a different module?
import soundcloud
import pandas as pd
from pandas import DataFrame
import MySQLdb
client =
soundcloud.Client(client_id='696b5ca70f5401cc46c9011c78831877')
userId = '110652450'
tracks = client.get('/users/'+userId+'/tracks')
data = []
for x in tracks:
data.append({'Track_Name':x.title,'plays':str(x.playback_count)})
df = pd.DataFrame(data)
database = MySQLdb.connect (host="127.0.0.1",user ="root",passwd="XXX",db="soundcloudstore")
cursor = database.cursor()
query = """INSERT INTO Tracks (Track_Name, Plays) VALUES (%s,%s)"""
for x in df:
Track_Name = df[['Track_Name']].value
Plays = df[['plays']].value
values = (Track_Name, Plays)
cursor.execute(query, values)
cursor.close()
database.commit()
database.close()
Download the adapter here: https://dev.mysql.com/doc/connector-python/en/connector-python-installation.html
Then you would use it as so:
import mysql.connector
data = []
for x in tracks:
data.append((x.title, str(x.playback_count)))
conn = mysql.connector.connect(user='', password='',
host='',
database='')
cursor = conn.cursor()
q = """INSERT INTO Tracks (Track_Name, Plays) VALUES (%s,%s)"""
cursor.executemany(q, data)
This will save you from loading into a dataframe for no reason and executemany is optimized for inserts.
Any help on this problem will be greatly appreciated.
So basically I want to run a query to my SQL database and store the returned data as Pandas data structure.
I have attached code for query.
I am reading the documentation on Pandas, but I have problem to identify the return type of my query.
I tried to print the query result, but it doesn't give any useful information.
Thanks!!!!
from sqlalchemy import create_engine
engine2 = create_engine('mysql://THE DATABASE I AM ACCESSING')
connection2 = engine2.connect()
dataid = 1022
resoverall = connection2.execute("
SELECT
sum(BLABLA) AS BLA,
sum(BLABLABLA2) AS BLABLABLA2,
sum(SOME_INT) AS SOME_INT,
sum(SOME_INT2) AS SOME_INT2,
100*sum(SOME_INT2)/sum(SOME_INT) AS ctr,
sum(SOME_INT2)/sum(SOME_INT) AS cpc
FROM daily_report_cooked
WHERE campaign_id = '%s'",
%dataid
)
So I sort of want to understand what's the format/datatype of my variable "resoverall" and how to put it with PANDAS data structure.
Here's the shortest code that will do the job:
from pandas import DataFrame
df = DataFrame(resoverall.fetchall())
df.columns = resoverall.keys()
You can go fancier and parse the types as in Paul's answer.
Edit: Mar. 2015
As noted below, pandas now uses SQLAlchemy to both read from (read_sql) and insert into (to_sql) a database. The following should work
import pandas as pd
df = pd.read_sql(sql, cnxn)
Previous answer:
Via mikebmassey from a similar question
import pyodbc
import pandas.io.sql as psql
cnxn = pyodbc.connect(connection_info)
cursor = cnxn.cursor()
sql = "SELECT * FROM TABLE"
df = psql.frame_query(sql, cnxn)
cnxn.close()
If you are using SQLAlchemy's ORM rather than the expression language, you might find yourself wanting to convert an object of type sqlalchemy.orm.query.Query to a Pandas data frame.
The cleanest approach is to get the generated SQL from the query's statement attribute, and then execute it with pandas's read_sql() method. E.g., starting with a Query object called query:
df = pd.read_sql(query.statement, query.session.bind)
Edit 2014-09-30:
pandas now has a read_sql function. You definitely want to use that instead.
Original answer:
I can't help you with SQLAlchemy -- I always use pyodbc, MySQLdb, or psychopg2 as needed. But when doing so, a function as simple as the one below tends to suit my needs:
import decimal
import pyodbc #just corrected a typo here
import numpy as np
import pandas
cnn, cur = myConnectToDBfunction()
cmd = "SELECT * FROM myTable"
cur.execute(cmd)
dataframe = __processCursor(cur, dataframe=True)
def __processCursor(cur, dataframe=False, index=None):
'''
Processes a database cursor with data on it into either
a structured numpy array or a pandas dataframe.
input:
cur - a pyodbc cursor that has just received data
dataframe - bool. if false, a numpy record array is returned
if true, return a pandas dataframe
index - list of column(s) to use as index in a pandas dataframe
'''
datatypes = []
colinfo = cur.description
for col in colinfo:
if col[1] == unicode:
datatypes.append((col[0], 'U%d' % col[3]))
elif col[1] == str:
datatypes.append((col[0], 'S%d' % col[3]))
elif col[1] in [float, decimal.Decimal]:
datatypes.append((col[0], 'f4'))
elif col[1] == datetime.datetime:
datatypes.append((col[0], 'O4'))
elif col[1] == int:
datatypes.append((col[0], 'i4'))
data = []
for row in cur:
data.append(tuple(row))
array = np.array(data, dtype=datatypes)
if dataframe:
output = pandas.DataFrame.from_records(array)
if index is not None:
output = output.set_index(index)
else:
output = array
return output
1. Using MySQL-connector-python
# pip install mysql-connector-python
import mysql.connector
import pandas as pd
mydb = mysql.connector.connect(
host = 'host',
user = 'username',
passwd = 'pass',
database = 'db_name'
)
query = 'select * from table_name'
df = pd.read_sql(query, con = mydb)
print(df)
2. Using SQLAlchemy
# pip install pymysql
# pip install sqlalchemy
import pandas as pd
import sqlalchemy
engine = sqlalchemy.create_engine('mysql+pymysql://username:password#localhost:3306/db_name')
query = '''
select * from table_name
'''
df = pd.read_sql_query(query, engine)
print(df)
MySQL Connector
For those that works with the mysql connector you can use this code as a start. (Thanks to #Daniel Velkov)
Used refs:
Querying Data Using Connector/Python
Connecting to MYSQL with Python in 3 steps
import pandas as pd
import mysql.connector
# Setup MySQL connection
db = mysql.connector.connect(
host="<IP>", # your host, usually localhost
user="<USER>", # your username
password="<PASS>", # your password
database="<DATABASE>" # name of the data base
)
# You must create a Cursor object. It will let you execute all the queries you need
cur = db.cursor()
# Use all the SQL you like
cur.execute("SELECT * FROM <TABLE>")
# Put it all to a data frame
sql_data = pd.DataFrame(cur.fetchall())
sql_data.columns = cur.column_names
# Close the session
db.close()
# Show the data
print(sql_data.head())
Here's the code I use. Hope this helps.
import pandas as pd
from sqlalchemy import create_engine
def getData():
# Parameters
ServerName = "my_server"
Database = "my_db"
UserPwd = "user:pwd"
Driver = "driver=SQL Server Native Client 11.0"
# Create the connection
engine = create_engine('mssql+pyodbc://' + UserPwd + '#' + ServerName + '/' + Database + "?" + Driver)
sql = "select * from mytable"
df = pd.read_sql(sql, engine)
return df
df2 = getData()
print(df2)
This is a short and crisp answer to your problem:
from __future__ import print_function
import MySQLdb
import numpy as np
import pandas as pd
import xlrd
# Connecting to MySQL Database
connection = MySQLdb.connect(
host="hostname",
port=0000,
user="userID",
passwd="password",
db="table_documents",
charset='utf8'
)
print(connection)
#getting data from database into a dataframe
sql_for_df = 'select * from tabledata'
df_from_database = pd.read_sql(sql_for_df , connection)
Like Nathan, I often want to dump the results of a sqlalchemy or sqlsoup Query into a Pandas data frame. My own solution for this is:
query = session.query(tbl.Field1, tbl.Field2)
DataFrame(query.all(), columns=[column['name'] for column in query.column_descriptions])
resoverall is a sqlalchemy ResultProxy object. You can read more about it in the sqlalchemy docs, the latter explains basic usage of working with Engines and Connections. Important here is that resoverall is dict like.
Pandas likes dict like objects to create its data structures, see the online docs
Good luck with sqlalchemy and pandas.
Simply use pandas and pyodbc together. You'll have to modify your connection string (connstr) according to your database specifications.
import pyodbc
import pandas as pd
# MSSQL Connection String Example
connstr = "Server=myServerAddress;Database=myDB;User Id=myUsername;Password=myPass;"
# Query Database and Create DataFrame Using Results
df = pd.read_sql("select * from myTable", pyodbc.connect(connstr))
I've used pyodbc with several enterprise databases (e.g. SQL Server, MySQL, MariaDB, IBM).
This question is old, but I wanted to add my two-cents. I read the question as " I want to run a query to my [my]SQL database and store the returned data as Pandas data structure [DataFrame]."
From the code it looks like you mean mysql database and assume you mean pandas DataFrame.
import MySQLdb as mdb
import pandas.io.sql as sql
from pandas import *
conn = mdb.connect('<server>','<user>','<pass>','<db>');
df = sql.read_frame('<query>', conn)
For example,
conn = mdb.connect('localhost','myname','mypass','testdb');
df = sql.read_frame('select * from testTable', conn)
This will import all rows of testTable into a DataFrame.
Long time from last post but maybe it helps someone...
Shorted way than Paul H:
my_dic = session.query(query.all())
my_df = pandas.DataFrame.from_dict(my_dic)
Here is mine. Just in case if you are using "pymysql":
import pymysql
from pandas import DataFrame
host = 'localhost'
port = 3306
user = 'yourUserName'
passwd = 'yourPassword'
db = 'yourDatabase'
cnx = pymysql.connect(host=host, port=port, user=user, passwd=passwd, db=db)
cur = cnx.cursor()
query = """ SELECT * FROM yourTable LIMIT 10"""
cur.execute(query)
field_names = [i[0] for i in cur.description]
get_data = [xx for xx in cur]
cur.close()
cnx.close()
df = DataFrame(get_data)
df.columns = field_names
pandas.io.sql.write_frame is DEPRECATED.
https://pandas.pydata.org/pandas-docs/version/0.15.2/generated/pandas.io.sql.write_frame.html
Should change to use pandas.DataFrame.to_sql
https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_sql.html
There is another solution.
PYODBC to Pandas - DataFrame not working - Shape of passed values is (x,y), indices imply (w,z)
As of Pandas 0.12 (I believe) you can do:
import pandas
import pyodbc
sql = 'select * from table'
cnn = pyodbc.connect(...)
data = pandas.read_sql(sql, cnn)
Prior to 0.12, you could do:
import pandas
from pandas.io.sql import read_frame
import pyodbc
sql = 'select * from table'
cnn = pyodbc.connect(...)
data = read_frame(sql, cnn)
best way I do this
db.execute(query) where db=db_class() #database class
mydata=[x for x in db.fetchall()]
df=pd.DataFrame(data=mydata)
If the result type is ResultSet, you should convert it to dictionary first. Then the DataFrame columns will be collected automatically.
This works on my case:
df = pd.DataFrame([dict(r) for r in resoverall])
Here is a simple solution I like:
Put your DB connection info in a YAML file in a secure location (do not version it in the code repo).
---
host: 'hostname'
port: port_number_integer
database: 'databasename'
user: 'username'
password: 'password'
Then load the conf in a dictionary, open the db connection and load the result set of the SQL query in a data frame:
import yaml
import pymysql
import pandas as pd
db_conf_path = '/path/to/db-conf.yaml'
# Load DB conf
with open(db_conf_path) as db_conf_file:
db_conf = yaml.safe_load(db_conf_file)
# Connect to the DB
db_connection = pymysql.connect(**db_conf)
# Load the data into a DF
query = '''
SELECT *
FROM my_table
LIMIT 10
'''
df = pd.read_sql(query, con=db_connection)