Develop Reference

psycopg2 AttributeError: 'Cursor' object has no attribute 'mogrify'

I have a class that helps me with SQL queries and inserts in Postgres. I'm using psycopg2==2.7.5 right now. One of the methods I'm using looks like this:
import pandas as pd
import psycopg2.extensions as ps_ext
from typing import List
def insert_with_open_connection(self, df: pd.DataFrame, table_name: str, cursor: ps_ext.cursor,
conn: ps_ext.connection,
success_msg: str = 'Success',
conflict_cols: List[str] = None):
try:
# Format the INSERT SQL query
cols = str(tuple(df.columns)).replace("'", '')
nc = df.shape[1]
ss = "(" + ''.join('%s,' for _ in range(nc))[:-1] + ")"
try:
args_str = str(b','.join(cursor.mogrify(ss, x) for x in df.values), 'utf-8')
except psycopg2.ProgrammingError:
args_str = str(b','.join(cursor.mogrify(ss, x) for x in self.clean_numpy_int_for_mogrify(df.values)),
'utf-8')
args_str = args_str.replace("\'NaN\'::float", 'NULL')
insert_sql = f'INSERT INTO {table_name} {cols} VALUES {args_str}'
if conflict_cols is not None:
conf_cols = str(tuple(conflict_cols)).replace("'", '').replace(',)', ')')
insert_sql += f"\nON CONFLICT {conf_cols} DO NOTHING"
insert_sql += ';'
cursor.execute(insert_sql)
conn.commit()
return success_msg, 200
except Exception:
return traceback.format_exc(), 400
The conn and cursor parameters are generated from a SqlAlchemy Engine with code like this:
def create_pool(self, **db_config):
db_user = self.user
db_pass = self.password
db_name = self.database
# e.g. "/cloudsql"
db_socket_dir = os.environ.get("DB_SOCKET_DIR", "/cloudsql")
# i.e "<PROJECT-NAME>:<INSTANCE-REGION>:<INSTANCE-NAME>"
cloud_sql_connection_name = os.environ.get("CLOUD_SQL_CONNECTION_NAME",
'<PROJECT-NAME>:<INSTANCE-REGION>:<INSTANCE-NAME>')
self.pool = sqlalchemy.create_engine(
# Equivalent URL:
# postgresql+pg8000://<db_user>:<db_pass>#/<db_name>
# ?unix_sock=<socket_path>/<cloud_sql_instance_name>/.s.PGSQL.5432
sqlalchemy.engine.url.URL.create(drivername="postgresql+pg8000",
username=db_user, # e.g. "my-database-user"
password=db_pass, # e.g. "my-database-password"
database=db_name, # e.g. "my-database-name"
query={"unix_sock":
f"{db_socket_dir}/{cloud_sql_connection_name}/.s.PGSQL.5432"}),
**db_config
)
def get_db_connection(self) -> Connection:
if self.pool is None:
self.create_pool()
assert isinstance(self.pool, Engine)
try:
return self.pool.raw_connection()
except psycopg2.OperationalError:
self.create_pool()
return self.pool.raw_connection()
#contextlib.contextmanager
def db_connect(self):
db = self.get_db_connection()
cur = db.cursor()
try:
yield db, cur
finally:
db.close()
I'm trying to use this code inside a Google Cloud Function (Linux) and I get the following error/traceback when I run the insert_with_open_connection method there:
Traceback (most recent call last):
File "/workspace/db/sql_helper.py", line 221, in insert_with_open_connection
args_str = str(b','.join(cursor.mogrify(ss, x) for x in df.values), 'utf-8')
File "/workspace/db/sql_helper.py", line 221, in <genexpr>
args_str = str(b','.join(cursor.mogrify(ss, x) for x in df.values), 'utf-8')
AttributeError: 'Cursor' object has no attribute 'mogrify'
It's obvious that the cursor in the code doesn't seem to have the attribute mogrify, but based on the docs here, the mogrify method should exist.

I took a look at the code and noticed that you were using import psycopg2.extensions as ps_ext; and clearly that had mogrify in
the docs.
Then I came across this line:
self.pool = sqlalchemy.create_engine(
# Equivalent URL:
# postgresql+pg8000://<db_user>:<db_pass>#/<db_name>
# ?unix_sock=<socket_path>/<cloud_sql_instance_name>/.s.PGSQL.5432
sqlalchemy.engine.url.URL.create(drivername="postgresql+pg8000",
username=db_user, # e.g. "my-database-user"
password=db_pass, # e.g. "my-database-password"
database=db_name, # e.g. "my-database-name"
query={"unix_sock":
f"{db_socket_dir}/{cloud_sql_connection_name}/.s.PGSQL.5432"}),
**db_config
)
You aren't using the psycopg2 driver; but the pg8000 one and tracing
the way things are generated, the cursor as returned by the db.cursor() which in turn was created by the self.pool.raw_connection(),
I came to the conclusion that the cursor wasn't a ps_ext cursor but
a pg8000 cursor, which doesn't have the mogrify method as
shown in: https://github.com/tlocke/pg8000/blob/main/pg8000/dbapi.py
This is the likelihood of why you're having this error. I think
the solution is to change to using psycopg2 driver instead.
That said, this answer could be wrong and I'm barking up the wrong tree.

Pandas dataframe: No numeric data to plot

I have a table stored in a database in MySQL.
I fetched the results using MySQL connector and copied it to DataFrame. There are no problems till that.
Now as I'm taking the results from MySQL, they are in the form of strings, so I've converted the strings to int for the values of CONFIRMED_CASES, and leave the STATE_NAME as str.
Now I want to plot it in a bar graph, with Numeric data as CONFIRMED_CASES and state names as STATE_NAME but it shows me this error:
Traceback (most recent call last):
File "c:\Users\intel\Desktop\Covid Tracker\get_sql_data.py", line 66, in <module>
fd.get_graph()
File "c:\Users\intel\Desktop\Covid Tracker\get_sql_data.py", line 59, in get_graph
ax = df.plot(y='STATE_NAME', x='CONFIRMED_CASES',
...
raise TypeError("no numeric data to plot")
TypeError: no numeric data to plot
Here's my code:
from operator import index
from sqlalchemy import create_engine
import mysql.connector
import pandas as pd
import matplotlib.pyplot as plt
mydb = mysql.connector.connect(
host="localhost",
user="abhay",
password="1234",
database="covid_db"
)
mycursor = mydb.cursor()
class fetch_data():
def __init__(self):
...
def get_data(self, cmd):
...
def get_graph(self):
mydb = mysql.connector.connect(
host="localhost",
user="abhay",
password="1234",
database="covid_db"
)
mycursor = mydb.cursor(buffered=True)
mycursor.execute(
"select CONFIRMED_CASES from india;")
query = mycursor.fetchall()
# the data was like 10,000 so I removed the ',' and converted it to int
query = [int(x[0].replace(',', '')) for x in query]
print(query)
query2 = mycursor.execute(
"select STATE_NAME from india;")
query2 = mycursor.fetchall()
# this is the query for state name and i kept it as str only...
query2 = [x[0] for x in query2]
print(query2)
df = pd.DataFrame({
'CONFIRMED_CASES': [query],
'STATE_NAME': [query2],
})
# it gives me the error here...
ax = df.plot(y='STATE_NAME', x='CONFIRMED_CASES',
kind='bar')
ax.ticklabel_format(style='plain', axis='y')
plt.show()
fd = fetch_data()
fd.get_graph()
I don't know why there is no numeric value. I set it to int, but still...

Your dataframe is defined using lists of a list. query and query2 are lists already. Define your dataframe as:
df = pd.DataFrame({
'CONFIRMED_CASES': query, # no brackets
'STATE_NAME': query2, # no brackets
})

Use below code for converting data to integer before plotting graph.
df['CONFIRMED_CASES'] = df['CONFIRMED_CASES'].astype('int64')

Heyy guys,
It's solved.
I used query.values and it worked...

how to connect mysql database in python using pymysql,KEY ERROR

im pretty new with python,im sorry if my question and my english doesnt sound right at all.
ive been trying to integrated my scraping output to mysql,but unfortunately i got stuck.can you good people help me?
the output/error
*<pymysql.cursors.Cursor object at 0x000000E81FF095E0>
<pymysql.connections.Connection object at 0x000000E81FEF9070>
KeyError Traceback (most recent call last)
in
9 data = csv.DictReader(scrapeddata)
10 for row in data:
---> 11 sql = "INSERT INTO emas ( tanggal, terakhir, pembukaan, tertinggi, terendah, vol, perubahan%) VALUES ('%s','%s','%s','%s','%s','%s','%s')" %(str(row["tanggal"]),str(row["terakhir"]),str(row["pembukaan"]),str(row["tertinggi"]),str(row["terendah"]),str(row["vol"]),str(row["perubahan%"]))
12 print(sql)
13 cur.execute(sql)
KeyError: 'tanggal'*
this is my code
import csv
import pymysql
#Connecting to MySQL in Windows
conn = pymysql.connect(host="127.0.0.1", port = 3306, user = "root", passwd = '', database = "mysql",
charset = "utf8")
cur = conn.cursor()
cur.execute("USE historis")
print(cur)
print(conn)
with open(r'C:\Users\shulhan\output_emas.csv') as scrapeddata:
data = csv.DictReader(scrapeddata)
for row in data:
sql = "INSERT INTO emas ( tanggal, terakhir, pembukaan, tertinggi, terendah, vol, perubahan%)
VALUES ('%s','%s','%s','%s','%s','%s','%s')"
%(str(row["tanggal"]),str(row["terakhir"]),str(row["pembukaan"]),str(row["tertinggi"]),str(row["terendah"]),str(row["vol"]),str(row["perubahan%"]))
print(sql)
cur.execute(sql)
conn.commit()

i think your connection setup should look like this:
connection = pymysql.connect(host='localhost',
user='user',
password='passwd',
db='db',
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor)
you can find more information here:
https://pymysql.readthedocs.io/en/latest/user/examples.html

Python/pyodbc not picking up raiserror or throw from SQL Server try/catch block?

I am expecting python/pyodbc to handle a raiserror or throw from a block of tsql (SQL Server). Instead, it appears to be returning a zero length list with no column information. Is this a driver issue or is there something that I am missing? MRE below...
import pyodbc
import pandas.io.sql as pd
from pandas import DataFrame
from tabulate import tabulate
import sys
import os
server = "XXX.XXX.XXX.XXX"
database = "XXXXXXXXXXXX"
username = "XXXXXXXXXXXX"
password = "XXXXXXXXXXXX"
try:
cnxn = pyodbc.connect('DRIVER={ODBC Driver 17 for SQL Server};MARS_Connection=Yes;SERVER='+server+';DATABASE='+database+';UID='+username+';PWD='+password)
# autocommit defaults to false on pyodbc but we add it here for absence of doubt.
cnxn.autocommit = False
cursor = cnxn.cursor()
#query_str = open(sys.argv[1],"r").read()
#query_str = "set nocount on; create table #test (col1 int, col2 varchar(10)); insert into #test (col1, col2) values (999, 'hello'); select * from #test"
query_str = """
set nocount on;
begin try
--create table #test (col1 int, col2 varchar(10));
--insert into #test (col1, col2) values (999, 'hello');
--select * from #test;
select 1/ 0 ;
end try
begin catch
raiserror('Hello from the db', 16, 1);
--throw;
end catch
"""
#print (query_str)
try:
retval = cursor.execute(query_str)
if cursor.description is not None:
columns = [column[0] for column in cursor.description]
rt = retval.fetchall()
detupled_rt = map(list, rt)
df = pd.DataFrame(detupled_rt)
table = df.values.tolist()
print(tabulate(table, columns, tablefmt="github"))
else:
print("Completed with no result set")
except pyodbc.ProgrammingError as pex:
print("Programming error")
raise pex
except pyodbc.Error as ex:
#sqlstate = ex.args[1]
print ("Program error")
raise ex
finally:
cnxn.commit()
cursor.close()
cnxn.close()
This works as I would expect if I remove the try/catch blocks from the sql query, for example...
query_str = """
set nocount on;
select 1/ 0 ;
"""
Expected output...
Program error
Traceback (most recent call last):
File "t_try.py", line 56, in <module>
raise ex
File "t_try.py", line 38, in <module>
retval = cursor.execute(query_str)
pyodbc.DataError: ('22012', '[22012] [Microsoft][ODBC Driver 17 for SQL Server][SQL Server]Divide by zero error encountered. (8134) (SQLExecDirectW)')

AttributeError: 'Engine' object has no attribute 'conn'

So I am trying to create an auto update to SQL from another excel file, by unique value, as to know what is the new data to add to the database..
There's different in columns names between the database and the excel file as in the database and names without spaces...
I tried to do it with pandas it gave me the same error
So here's my simple code tried with xlrd
import xlrd
from sqlalchemy import create_engine
def insert():
book = xlrd.open_workbook(r"MNM_Rotterdam_5_Daily_Details-20191216081027 - Copy (2).xlsx")
sheet = book.sheet_by_name("GSM Details")
database = create_engine(
'mssql+pyodbc://WWX542337CDCD\SMARTRNO_EXPRESS/myDB?driver=SQL+Server+Native+Client+11.0') # name of database
cnxn = database.raw_connection
cursor = cnxn.cursor()
query = """Insert INTO [myDB].[dbo].[mnm_rotterdam_5_daily_details-20191216081027] (Date, SiteName, CellCI, CellLAC, CellName, CellIndex) values (?,?,?,?,?,?)"""
for r in range(1, sheet.nrows):
date = sheet.cell(r,0).value
site_name = sheet.cell(r,3).value
cell_ci = sheet.cell(r,4).value
cell_lac = sheet.cell(r,5).value
cell_name = sheet.cell(r,6).value
cell_index = sheet.cell(r,7).value
values = (date, site_name, cell_ci, cell_lac, cell_name, cell_index)
cursor.execute(query, values)
cnxn.commit()
# Close the cursor
cursor.close()
# Commit the transaction
database.commit()
# Close the database connection
database.close()
# Print results
print ("")
print ("")
columns = str(sheet.ncols)
rows = str(sheet.nrows)
print ("Imported", columns,"columns and", rows, "rows. All Done!")
insert()
and this is the error:
I tried to change the range I found another error:
Traceback (most recent call last):
File "D:/Tooling/20200207/uniquebcon.py", line 48, in <module>
insert()
File "D:/Tooling/20200207/uniquebcon.py", line 37, in insert
database.commit()
AttributeError: 'Engine' object has no attribute 'commit'
I think this is related to SQL-Alchemy in the connection

Instead of creating the cursor directly with
cursor = database.raw_connection().cursor()
you can create a connection object, then create the cursor from that, and then call .commit() on the connection:
cnxn = database.raw_connection()
crsr = cnxn.cursor()
# do stuff with crsr ...
cnxn.commit()