I wasn't sure what to call this, feel free to edit my post title.
Before I begin, I googled and looked here, but this didn't seem to help me.
My code:
import pyodbc
import pandas as pd
import numpy as np
import os
import sqlalchemy as sal
from sqlalchemy import create_engine
from sqlalchemy import MetaData
from sqlalchemy import Table, Column, Integer, Numeric, String, ForeignKey, Boolean
##
from datetime import datetime
from sqlalchemy import DateTime
from sqlalchemy import PrimaryKeyConstraint, UniqueConstraint, CheckConstraint
from sqlalchemy import Index
from sqlalchemy import ForeignKeyConstraint
from sqlalchemy import insert
from sqlalchemy.sql import select
from sqlalchemy.sql import func
from sqlalchemy import cast
from sqlalchemy import and_, or_, not_
from sqlalchemy import update, delete
from sqlalchemy import text
##
import urllib
#############################################################
server = 'fake_server'
database = '_Testing_Only'
driver = 'SQL+SERVER+NATIVE+CLIENT+11.0'
trusted_connection='yes'
database_connection = 'mssql+pyodbc://fake_server/' + database + '?trusted_connection=' + trusted_connection + '&driver=' + driver
engine = sal.create_engine(database_connection)
connection=engine.connect()
metadata = MetaData()
print(engine.table_names())
Here is the result of my print statement:
['cookies', 'line_items', 'orders', 'testing_sym_keys', 'users']
I then tried to run this code:
s = select([cookies])
I got the following error message:
Traceback (most recent call last):
File "<pyshell#167>", line 1, in <module>
s = select([cookies])
NameError: name 'cookies' is not defined
The table clearly exists, why am I getting the error message?
The issue is that you are not binding tables in the engine variable to a variable in your Python code.
Trying running a native SQL query of the form:
engine = create_engine(database_connection)
metadata = MetaData(engine)
metadata.reflect()
with engine.begin() as conn:
conn.execute("select * from cookies")
Other solution, if you want to use select method you can try this:
engine = create_engine(database_connection)
meta = MetaData(engine).reflect()
table = meta.tables['cookies']
# select * from 'cookies'
select_cookies = select([table])
Just found a fast solution for this. Try creating table object first for your desired table.
engine = create_engine(database_connection)
metadata = MetaData()
cookies = Table('cookies', metadata, autoload=True, autoload_with=engine)
So everytime you want to query a table, just do the above code each one so that you initialize the table in python.
Related
I am trying to drop a table called 'New'. I currently have the following code:
import pandas as pd
import sqlalchemy
sqlcon = sqlalchemy.create_engine('mssql://ABSECTDCS100TL/AdventureWorks?driver=ODBC+Driver+17+for+SQL+Server')3
df = pd.read_sql_query('SELECT * FROM DimReseller', sqlcon)
df.to_sql('New',sqlcon,if_exists='append', index=False)
sqlalchemy.schema.New.drop(bind=None, checkfirst=False)
I am receiving the error:
AttributeError: module 'sqlalchemy.schema' has no attribute 'New'
Any ideas on what I'm missing here?. Thanks.
You can reflect the table into a Table object and then call its drop method:
from sqlalchemy import Table, MetaData
tbl = Table('New', MetaData(), autoload_with=sqlcon)
tbl.drop(sqlcon, checkfirst=False)
If you want to delete the table using raw SQL, you can do this:
from sqlalchemy import text
with sqlcon.connect() as conn:
# Follow the identifier quoting convention for your RDBMS
# to avoid problems with mixed-case names.
conn.execute(text("""DROP TABLE "New" """))
# Commit if necessary
conn.commit()
I cannot find out, how to use pandas.read_sql_query and correctly (= safely against sql injection) parametrize table names (or other sql identifiers). Using sqlalchemy+psycopg2 to access PostgreSQL database.
Example of what doesn't work:
import os
import pandas
from sqlalchemy import create_engine
db = create_engine(os.getenv(POSTGRES_CONNSTRING))
pandas.read_sql_query(sql='select * from %(schema)s.%(table)s',
con = db,
params={'schema': 'public', 'table': 'table_name'})
Yields:
SyntaxError: syntax error at or near "'public'"
LINE 1: select * from 'public'.'table_name'
For psycopg2 the correct solution is described here.
import psycopg2
query = psycopg2.sql.SQL('select * from {schema}.{table}') \
.format(schema = psycopg2.sql.Identifier('public'),
table = psycopg2.sql.Identifier('table_name'))
But the query is now of type psycopg2.sql.Composed, which I can pass to the execute methods in psycopg2 but not to pandas.read_sql_query.
Is there any good solution to this?
You can use the as_string method to turn the Composed query into a string that you can pass to Pandas (docs).
import pandas as pd
import psycopg2
from sqlalchemy import create_engine
engine = create_engine('postgresql+psycopg2://user:pw#host:port/db')
cur = engine.raw_connection().cursor()
query = psycopg2.sql.SQL('select * from {schema}.{table}') \
.format(schema = psycopg2.sql.Identifier('public'),
table = psycopg2.sql.Identifier('table_name'))
query_string = query.as_string(cur)
pd.read_sql_query(query_string, engine)
I'm having issues importing data with python into a table on my Database directly from an excel file.
I have this code:
import os
import pandas as pd
import pymysql
if os.path.exists("env.py"):
import env
print(os.environ)
# Abre conexion con la base de datos
db = pymysql.connect(
host = os.environ.get("MY_DATABASE_HOST"),
user = os.environ.get("MY_USERNAME"),
password = os.environ.get("MY_PASSWORD"),
database = os.environ.get("MY_DATABASE_NAME")
)
##################################################
################LECTURA DE EXCEL##################
tabla_azul = "./excelFiles/tablaAzul.xlsx"
dAzul = pd.read_excel(tabla_azul, sheet_name='Órdenes')
dAzul.to_sql(con=db, name='tablaazul', if_exists='append', schema='str')
#print(type(dAzul))
tabla_verde = "./excelFiles/tablaVerde.xlsx"
dVerde = pd.read_excel(tabla_verde, sheet_name='Órdenes')
dVerde.to_sql(con=db, name='tablaverde', if_exists='append', schema='str')
I'm not sure what table name I have to put into the name variable.
Do I need to use sqlalchemy yes or yes?
If question 2 is yes: Is it possible to connect sqlalchemy with pymysql?
If question 3 is no: Ho do I use the .env variables like host with sqlalchemy connection?
thank you!
when I run the code above, it's giving me this error:
pandas.io.sql.DatabaseError: Execution failed on sql 'SELECT name FROM sqlite_master WHERE type='table' AND name=?;': not all arguments converted during string formatting
As stated in the pandas documentation, for any database other than SQLite .to_sql() requires a SQLAlchemy Connectable object, which is either an Engine object or a Connection object. You can create an Engine object for PyMySQL like so:
import sqlalchemy as sa
connection_url = sa.engine.URL.create(
"mysql+pymysql",
username=os.environ.get("MY_USERNAME"),
password=os.environ.get("MY_PASSWORD"),
host=os.environ.get("MY_DATABASE_HOST"),
database=os.environ.get("MY_DATABASE_NAME")
)
engine = sa.create_engine(connection_url)
Then you can call .to_sql() and pass it the engine:
dVerde.to_sql(con=engine, name='tablaverde', if_exists='append', schema='str')
Using SQLAlchemy ORM, I would like to get the explain plan for an Oracle query.
This would be done in sqlplus using 2 steps that look something like this:
# step 1
explain plan for select * from table_xyz;
# step 2
select * from table(dbms_xplan.display(null, null, 'SERIAL'));
I have tried the following so far with SQLAlchemy ORM without luck:
from sqlalchemy import create_engine, Table
from sqlalchemy.orm import sessionmaker
import os
# create engine
user, pswd = (os.environ['USER'], os.environ['PW'])
conn_str = f'oracle://{user}:{pswd}#myDbService'
engine = create_engine(conn_str, echo=True)
# create session
Session = sessionmaker(bind=engine)
# reflect existing table
MyTable = Table('my_table', meta, autoload=True, autoload_with=engine, schema='myschema')
# generate query
bind_param = dict('state_cd'=['CA', 'WA'])
query = str(Session().query(MyTable).filter(MyTable.c.my_state_cd.in_(state_cd)))
# print(query) # <-- this returns a properly formulated select query with bound parameters
result = Session().execute('EXPLAIN PLAN FOR ' + query, bind_param)
Executing the last line above keeps failing with the following error and I'm not sure what I'm doing wrong:
StatementError: (sqlalchemy.exc.InvalidRequestError) A value is required for bind parameter 'my_state_cd_2'
Is it possible to convert retrieved SqlAlchemy table object into Pandas DataFrame or do I need to write a particular function for that aim ?
This might not be the most efficient way, but it has worked for me to reflect a database table using automap_base and then convert it to a Pandas DataFrame.
import pandas as pd
from sqlalchemy.ext.automap import automap_base
from sqlalchemy import create_engine
from sqlalchemy.orm import Session
connection_string = "your:db:connection:string:here"
engine = create_engine(connection_string, echo=False)
session = Session(engine)
# sqlalchemy: Reflect the tables
Base = automap_base()
Base.prepare(engine, reflect=True)
# Mapped classes are now created with names by default matching that of the table name.
Table_Name = Base.classes.table_name
# Example query with filtering
query = session.query(Table_Name).filter(Table_Name.language != 'english')
# Convert to DataFrame
df = pd.read_sql(query.statement, engine)
df.head()
I think I've tried this before. It's hacky, but for whole-table ORM query results, this should work:
import pandas as pd
cols = [c.name for c in SQLA_Table.__table__.columns]
pk = [c.name for c in SQLA_Table.__table__.primary_key]
tuplefied_list = [(getattr(item, col) for col in cols) for item in result_list]
df = pd.DataFrame.from_records(tuplefied_list, index=pk, columns=cols)
Partial query results (NamedTuples) will also work, but you have to construct the DataFrame columns and index to match your query.
Pandas database functions such as read_sql_query accept SQLAlchemy connection objects (so-called SQLAlchemy connectables, see pandas docs and sqlalchemy docs). Here's one example of using such object called my_connection:
import pandas as pd
import sqlalchemy
# create SQLAlchemy Engine object instance
my_engine = sqlalchemy.create_engine(f"{dialect}+{driver}://{login}:{password}#{host}/{db_name}")
# connect to the database using the newly created Engine instance
my_connection = my_engine.connect()
# run SQL query
my_df = pd.read_sql_query(sql=my_sql_query, con=my_connection)
I have a simpler way:
# Step1: import
import pandas as pd
from sqlalchemy import create_engine
# Step2: create_engine
connection_string = "sqlite:////absolute/path/to/database.db"
engine = create_engine(connection_string)
# Step3: select table
print (engine.table_names())
# Step4: read table
table_df = pd.read_sql_table('table_name', engine)
table_df.head()
For other types of connection_string, SQLAlchemy 1.4 Documentation.