I can successfully connect to SQL Server Management Studio from my jupyter notebook with this script:
from sqlalchemy import create_engine
import pyodbc
import csv
import time
import urllib
params = urllib.parse.quote_plus('''DRIVER={SQL Server Native Client 11.0};
SERVER=SV;
DATABASE=DB;
TRUSTED_CONNECTION=YES;''')
engine = create_engine("mssql+pyodbc:///?odbc_connect=%s" % params)
I managed to execute some SQL scripts like this:
engine.execute("delete from table_name_X")
However, I can't execute stored procedures. I tried the following scripts from what I've seen in stored procedures with sqlAlchemy. These following scripts have an output like "sqlalchemy.engine.result.ResultProxy at 0x173ed18e470", but the procedure wasn't executed in reality (nothing happened):
# test 1
engine.execute('stored_procedure_name')
# test 2
from sqlalchemy import func
from sqlalchemy.orm import sessionmaker
session = sessionmaker(bind=engine)()
session.execute(func.upper('stored_procedure_name'))
Could you please give me the correct way to execute stored procedures?
The way you can call a stored procedure using pyodbc is :
cursor.execute("{CALL usp_StoreProcedure}")
I found a solutions in reference to this link . https://github.com/mkleehammer/pyodbc/wiki/Calling-Stored-Procedures
Here a example :
import pyodbc
import urllib
import sqlalchemy as sa
params = urllib.parse.quote_plus("DRIVER={SQL Server Native Client 11.0};"
"SERVER=xxx.xxx.xxx.xxx;"
"DATABASE=DB;"
"UID=user;"
"PWD=pass")
engine = sa.create_engine("mssql+pyodbc:///?odbc_connect={}".format(params))
connection = engine.raw_connection()
try:
cursor = connection.cursor()
cursor.execute("{CALL stored_procedure_name}")
result = cursor.fetchall()
print(result)
connection.commit()
finally:
connection.close()
Finally solved my problem with the following function :
def execute_stored_procedure(engine, procedure_name):
res = {}
connection = engine.raw_connection()
try:
cursor = connection.cursor()
cursor.execute("EXEC "+procedure_name)
cursor.close()
connection.commit()
res['status'] = 'OK'
except Exception as e:
res['status'] = 'ERROR'
res['error'] = e
finally:
connection.close()
return res
Related
I have Python script that I am trying to use to execute this function below on my SQL Server
DBCC CHECKIDENT('TableName', RESEED, 0)
My script looks like this:
qry = '''DBCC CHECKIDENT('TableName', RESEED, 0)'''
def mssql_cmd(qry, env):
# Import Dependencies
import pyodbc
import sqlalchemy as sa
import urllib
import pandas as pd
import sqlalchemy
import json
try:
# Read config json file into config dict
with open("../parameters/config.json") as cf:
config = json.load(cf)
# Try to establish the connection to MSSQL
params = urllib.parse.quote_plus(f'DRIVER={config[env][0]["driver"]};'
f'Server={config[env][0]["server"]};'
f'Database={config[env][0]["database"]};'
f'User={config[env][0]["user"]};'
f'Password={config[env][0]["password"]};'
f'Trusted_connection={config[env][0]["Trusted_connection"]};')
# Establish the engine
engine = sa.create_engine("mssql+pyodbc:///?odbc_connect=%s" % params)
db = engine.connect()
print("Connection to Data Warehouse -- SUCCESSFUL")
if db.connect():
try:
db.execute(qry)
db.close()
engine.dispose()
except Exception as e:
print(e)
except Exception as e:
print(e)
I don't get any errors the scrips executes but it doesn't reset my autogen Id on the table.
If I replace the line
db.execute(qry)
with
data = pd.read_sql(sql_qry, db)
then I am able to extract the data.
So the script works if I run the query however I can't make it to run the function to reset my auto gen id.
Does anyone have any clue as to what I am doing wrong here?
I am trying to load the data from the REST API into SQL server table using the Python script. The script below works perfectly but the issues is it takes too long to upload the data in to the database , it took few minutes to load 8000 records
import requests
import pandas as pd
import pyodbc
import sys
from requests.auth import HTTPBasicAuth
#SQL Connection
conn_str = (
r'DRIVER={SQL Server Native Client 11.0};'
r'SERVER=tcp:abcd.database.windows.net;'
r'DATABASE=DEF;'
r'UID=xxxxx;'
r'PWD=yyyy;'
r'Trusted_Connection=no;'
)
sqlconn = pyodbc.connect(conn_str)
cursor = sqlconn.cursor()
cursor.execute("TRUNCATE TABLE [dbo].[RM_Approved_Room_State]")
sqlconn.commit()
#API Connection
baseURL = 'https://testing.org/api/fdf'
appRoomsResponse = requests.get(baseURL, verify=False)
appRoomsJson = appRoomsResponse.json()
appRoomsDF = pd.json_normalize(appRoomsJson)
appRoomsDF = appRoomsDF.fillna('')
try:
cursor = sqlconn.cursor()
for index,row in appRoomsDF.iterrows():
cursor.execute("INSERT INTO RM_Approved_Room_State(APPROVED_ROOM_STATEID,SOURCE_ROOMID,DEST_ROOMID,ENTITY_TYPEID)\
values(?,?,?,?)"
,row['id']
,row['sourceRoomRefId']
,row['destinationRoomRefId']
,row['entityRefId']
)
sqlconn.commit()
except Exception:
pass
#Load main Dim_appRooms table
cursor = sqlconn.cursor()
sqlconn.commit()
cursor.close()
sqlconn.close()
Is there something I am missing to increase the speed of the insert here. My first script in Python, anyhelp is greatly appreciated
hi all im trying to run a stored procedure using python however im getting an error below is my code
import pyodbc
import sqlalchemy
from urllib.parse import quote_plus
import sqlalchemy as sa
driver='SQL Server Native Client 11.0'
params = quote_plus(r'DRIVER={SQL Server Native Client 11.0};'
r'SERVER=i cant disclose my server ;'
r'DATABASE=ForTestPurpose;'
r'Uid='+sql_userid+';'
r'Pwd='+sql_pwd+';')
engine = sqlalchemy.create_engine('mssql+pyodbc:///?odbc_connect=%s' % params)
query= sa.text("EXEC InsertTestVisionOpenCases '"+yesterday1+"' ")
engine.execute(query)#you can try entering with parameters as how you give in your sql
engine.execution_options(autocommit=True)
cursor.close()
conn.close()
and im getting this error
ProgrammingError: Attempt to use a closed cursor.
please let me know how can i remove that error
I was able to solve the issue please check the below code it helped me to move ahead.
conn = pyodbc.connect(r'DRIVER={SQL Server Native Client 11.0};'
r'SERVER=W-MUM-myserver_name;'
r'DATABASE=Analytics;'
r'Uid='+sql_userid+';'
r'Pwd='+sql_pwd+';')
cursor = conn.cursor()
params=date(yesterday.year,yesterday.month,yesterday.day)
storedProc = "Exec ForTestPurpose.dbo.InsertThreeVisionOpenCases #parameter1 = ?"
cursor.execute( storedProc, params )
conn.commit()
cursor.close()
conn.close()
I need to access tables from Impala through CLI using python on the same cloudera server
I have tried below code to establish the connection :
def query_impala(sql):
cursor = query_impala_cursor(sql)
result = cursor.fetchall()
field_names = [f[0] for f in cursor.description]
return result, field_names
def query_impala_cursor(sql, params=None):
conn = connect(host='xx.xx.xx.xx', port=21050, database='am_playbook',user='xxxxxxxx', password='xxxxxxxx')
cursor = conn.cursor()
cursor.execute(sql.encode('utf-8'), params)
return cursor
but since I am on the same cloudera server, I will not need to provide the host name. Could you please provide the correct code to access Impala/hive tables existing on the same server through python.
you can use pyhive to make connection to hive and get access to your hive tables.
from pyhive import hive
import pandas as pd
import datetime
conn = hive.Connection(host="hostname", port=10000, username="XXXX")
hive.connect('hostname', configuration={'hive.execution.engine':'tez'})
query="select col1,col2,col3,col4 from db.yourhiveTable"
start_time= datetime.datetime.now()
data=pd.read_sql(query,conn)
print(data)
end_time=datetime.datetime.now()
print 'Finished reading from Hive table', (start_time-end_time).seconds/60.0,' minutes'
I'm new to hadoop and impala. I managed to connect to impala by installing impyla and executing the following code. This is connection by LDAP:
from impala.dbapi import connect
from impala.util import as_pandas
conn = connect(host="server.lrd.com",port=21050, database='tcad',auth_mechanism='PLAIN', user="alexcj", use_ssl=True,timeout=20, password="secret1pass")
I'm then able to grab a cursor and execute queries as:
cursor = conn.cursor()
cursor.execute('SELECT * FROM tab_2014_m LIMIT 10')
df = as_pandas(cursor)
I'd like to be able use sqlalchemy to connect to impala and be able to use some nice sqlalchemy functions. I found a test file in imyla source code that illustrates how to create an sqlalchemy engine with impala driver like:
engine = create_engine('impala://localhost')
I'd like to be able to do that but I'm not able to because my call to the connect function above has a lot more parameters; and I do not know how to pass those to sqlalchemy's create_engine to get a successful connection. Has anyone done this? Thanks.
As explained at https://github.com/cloudera/impyla/issues/214
import sqlalchemy
def conn():
return connect(host='some_host',
port=21050,
database='default',
timeout=20,
use_ssl=True,
ca_cert='some_pem',
user=user, password=pwd,
auth_mechanism='PLAIN')
engine = sqlalchemy.create_engine('impala://', creator=conn)
If your Impala is secured by Kerberos below script works (due to some reason I need to use hive:// instead of impala://)
import sqlalchemy
from sqlalchemy.engine import create_engine
connect_args={'auth': 'KERBEROS', 'kerberos_service_name': 'impala'}
engine = create_engine('hive://impalad-host:21050', connect_args=connect_args)
conn = engine.connect()
ResultProxy = conn.execute("SELECT * FROM db1.table1 LIMIT 5")
print(ResultProxy.fetchall())
import time
from sqlalchemy import create_engine, MetaData, Table, select, and_
ENGINE = create_engine(
'impala://{host}:{port}/{database}'.format(
host=host, # your host
port=port,
database=database,
)
)
METADATA = MetaData(ENGINE)
TABLES = {
'table': Table('table_name', METADATA, autoload=True),
}