class code:
def rowcount(self):
return self._psql_cur.rowcount
And the code in the main program:
def sql_query(self, sql, *data, **kwdata):
"""
NOTE: This function returns a generator. So if you use it to do any kind of update to the dbms that doesn't
return anything, it won't be executed!
"""
self.last_select_id += 1
n_retrials = kwdata.get("___n_retrials", 0)
if n_retrials > 10:
raise OperationalError
assert not (len(data) > 0 and len(set(kwdata) - {"___n_retrials"}) > 0), \
"Pass either keyword-based data or comma-separated data."
time_start = time.time()
n_records_retrieved = 0
status = None
toclose = False
print "*********************inside db.py******************"
if self.logfile is not None:
self.logfile.write(">>> {} {} {} START SELECT\n{}\ndata={}\nkwdata={}\n\n".format(
self.cursor_id, self.last_select_id, time_start, sql, data, kwdata))
print "\n************* QUERY:\n", sql
print "\n***************************"
try:
if len(data) > 0:
print "\n**************printing data***********",data
print "\n******************printing sql**************************",sql
print "\n*******************************************************"
# self._psql_cur.execute(sql, data)
cur, toclose = self._execute_query(sql, data)
elif len(kwdata) > 0:
# self._psql_cur.execute(sql, kwdata)
cur, toclose = self._execute_query(sql, kwdata)
else:
cur, toclose = self._execute_query(sql, None)
print "################check###################"
n_records_reported = cur.rowcount
print "\n*************printing rowcount**********",n_records_reported
# Yield records
for record in cur:
n_records_retrieved += 1
if n_records_retrieved == n_records_reported:
status = "Finished"
yield record
following code conatains _execute_query:
def _execute_query(self, sql, args):
# sql = sql.lower().strip()
# print sql
sql_strip = sql.lower().strip()
print "-------4",args
# print self.dbname, sql_strip
if sql_strip.startswith("select ") or \
(sql_strip.startswith("with ")
# and "update " not in sql_strip and "insert " not in sql_strip
):
# Try to close previous named cursor
# if self._psql_cur is not None and not self._psql_cur.closed:
# try:
# self._psql_cur.close()
# except ProgrammingError:
# pass
# self._psql_cur.scroll(self._psql_cur.rowcount, mode="absolute")
# self._psql_cur.fetchone()
# self._psql_cur.fetchone()
# Create a new named cursor
self._psql_cur = self.connection.get_cursor()
print self.dbname, "NAMED", self._psql_cur
# Execute query
self._psql_cur.execute(sql, args)
rows = self._psql_cur.fetchall()
print "FETCHED RESULT: ", rows
print sql
return rows, True
#self._psql_cur.execute("""select * from recipes""")
#rows=self._psql_cur.fetchall()
#print "---------------5 ",rows[0]
#self._psql_cur.fetchall()
return self._psql_cur, True
else:
# if "insert " in sql or "update " in sql or "delete " in sql or "create" in sql:
# print self.dbname, "UNNAMED"
# In this case, do not use the named (server side) cursor
# self._psql_unnamed_cur = self._connection.get_cursor(named=False)
self._psql_unnamed_cur.execute(sql, args)
return self._psql_unnamed_cur, False
I can't figure out why I'm getting this error.
I am trying to get data from database here actually .This is a part of code available in the Github.(PACKAGE QUERY). This is the output I am getting:
Exception occured while executing query:
File "src/dbms/db.py", line 378, in sql_query
n_records_reported = cur.rowcount
AttributeError: 'list' object has no attribute 'rowcount'
Exception during experiment
'list' object has no attribute 'rowcount'
Please tell if you need more information about this doubt. :-)
Your _execute_query method returns a list when your query starts with select or with:
if sql_strip.startswith("select ") or \
(sql_strip.startswith("with ")
# and "update " not in sql_strip and "insert " not in sql_strip
):
# ...
rows = self._psql_cur.fetchall()
print "FETCHED RESULT: ", rows
print sql
return rows, True
rows is a list, not a cursor, so won't have the attribute. Either return the cursor there, or use len() to get a count of rows.
Related
I am trying to populate an azure sql database using pyodbc.
I have around 14000item to push into the database per months, and running and executing each single upsert on it is really long.
I am so trying to run them in batch and push them all at once. Sadly, when doing this I am not getting anything in the database. Is there any way to make this works ?
i = 0
queryUpdate = ""
for values in valuesList:
if i == 100:
i = 0
queryUpdate = queryUpdate.replace('\'NULL\'', "NULL") #set null to real sql null
queryUpdate = queryUpdate.replace("True", "1") #replace true with 1 to match the bit type
queryUpdate = queryUpdate.replace("False", "0") #replace false with 0 to match the bit type
try:
cursor.execute(queryUpdate)
except pyodbc.Error as ex:
print(f"[x] Error when running the query:\n[x] {queryUpdate}\n[x] Exception: {ex}")
queryUpdate = ""
queryUpdate += f"""\n
if exists (select * from {tableName} with (updlock,serializable) where {colNames[keyIndex]} = {values[keyIndex]})
begin
update {tableName} set """
for i in range(len(colNames)):
queryUpdate += f"{colNames[i]} = '{values[i]}'"
if i != len(colNames) - 1:
queryUpdate += ","
queryUpdate += f"""
where {colNames[keyIndex]} = {values[keyIndex]}
end
else
begin
insert into {tableName} ({','.join(colNames)})
values {tuple(values)}
end;
"""
i+=1
try:
conn.commit()
except pyodbc.Error as ex:
print(f"[x] Error when commiting to the database.\n[x] Exception: {ex}")
else:
print("[+] Commit confirmed !")
cursor.close()
conn.close()
print("[+] Connection closed.")
I am trying to get rows quantity from query using python to mysql. I've been using rowcount but returned value is always 0.
def getLastData(md5TwitterDate):
conectar = connection()
try:
cursor = conectar.cursor()
query = "SELECT fecha,numeroreporte,internoid FROM jsywe_ncnsismos_sismosigp WHERE internoid = '%s' and published=1"
cursor.execute(query, md5TwitterDate)
lastEvent = cursor.fetchall()
rowsa = cursor.rowcount
except Error as ex:
print("Error to get data: ", ex)
finally:
if conectar.is_connected():
conectar.close()
#return False if rowsa> 0 else True
return rowsa
Also tried this way setting a variable to cursor.execute but in this case always get none
def getLastData(md5TwitterDate):
conectar = connection()
try:
cursor = conectar.cursor()
query = "SELECT fecha,numeroreporte,internoid FROM jsywe_ncnsismos_sismosigp WHERE internoid = '%s' and published=1"
rowsa = cursor.execute(query, md5TwitterDate)
lastEvent = cursor.fetchall()
except Error as ex:
print("Error to get data: ", ex)
finally:
if conectar.is_connected():
conectar.close()
#return False if filas > 0 else True
return rowsa
Tested query on database and it works, it returns 1 row
I have a database class which abstracts some basic crud logic.
The issue lies in the fetch_single method:
The sql_insecure query works fine, and returns the expected results.
The sql_prepared query doesn't return any errors, but also doesn't return any results which match the parameters, when they clearly do exist within the db.
sql_prepared follows the same approach to prepared statements that the insert_single method implements, and this method also returns the expected results.
My question is; why is the sql_prepared query not returning any results?
import sqlite3
class Database:
def __init__(self, db: str):
try:
self.conn = sqlite3.connect(db)
self.cursor = self.conn.cursor()
except sqlite3.Error as e:
print(e)
self.__del__
def fetch_all(self, table: str):
try:
query = self.cursor.execute("SELECT * FROM ?", table)
rows = self.cursor.fetchall()
return rows
except sqlite3.Error as e:
print(e)
return False
def fetch_single(self, table: str, column_name: str, column_value):
sql_formatted_value = "'{value}'".format(value=column_value)
placeholder = ":{column_name}".format(column_name=column_name)
sql_insecrue = "SELECT * FROM %s WHERE %s=%s Limit 1" % (
table, column_name, sql_formatted_value)
sql_prepared = "SELECT * FROM %s WHERE %s=%s LIMIT 1" % (
table, column_name, placeholder)
# try:
# self.cursor.execute(sql_insecrue)
# rows = self.cursor.fetchall()
# return rows
# except sqlite3.Error as e:
# print(e)
# return False
try:
self.cursor.execute(sql_prepared, [sql_formatted_value, ])
rows = self.cursor.fetchall()
return rows
except sqlite3.Error as e:
print(e)
return False
def insert_single(self, table: str, data: list):
columns = ""
placeholders = ""
values = []
data_length = len(data)
for index, (key, value) in enumerate(data):
# we need to dynamically build some strings based on the data
# let's generate some placeholders to execute prepared statements
columns += "{column_name}".format(column_name=key)
placeholders += ":{column_name}".format(column_name=key)
# let's fill the insert values into a list to use with execute
values.append(value)
# only add a comma if there is another item to assess
if index < (data_length - 1):
columns += ', '
placeholders += ', '
sql = "INSERT INTO %s (%s) VALUES (%s)" % (
table, columns, placeholders)
try:
self.cursor.execute(sql, values)
self.conn.commit()
except sqlite3.Error as e:
print(e)
You cannot substitute table name using ? in prepared statements because it is not considered a query parameter.
I recommend doing something like this:
self.cursor.execute(f"DELETE FROM {table} WHERE id=?", [id])
In other words, use standard python format statements to specify your table name, but use prepared statement anchors like ? for any query parameters.
okay, i found the problem.
its was my sloppy sql syntax.
using backticks around the table and column name solved the issue.
def fetch_single(self, table: str, column_name: str, column_value):
sql_formatted_value = "'{value}'".format(value=column_value)
placeholder = ":{column_name}".format(column_name=column_name)
sql_insecure = "SELECT * FROM %s WHERE %s=%s" % (
table, column_name, sql_formatted_value)
sql_prepared = "SELECT * FROM `%s` WHERE `%s`=%s" % (
table, column_name, placeholder)
print(sql_insecure)
print(sql_prepared)
# try:
# self.cursor.execute(sql_insecure)
# row = self.cursor.fetchall()
# print(row)
# return row
# except sqlite3.Error as e:
# print(e)
# return False
try:
self.cursor.execute(sql_prepared,
[column_value, ])
row = self.cursor.fetchone()
return row
except sqlite3.Error as e:
print(e)
return False
I'm trying to execute the same query but with different data but I always get data the first time. The others times, dispite of there are data for the querys in the data base, mysql returns empty data.
This is the code:
def get_team_colour_map(self, players, id_competition):
tcm = FIBAColourMap()
for p in players:
args = [p["id"], id_competition]
conn = pymysql.Connect(host = DDBB.DDBB_FIBA_HOST,
user = DDBB.DDBB_FIBA_USER,
password = DDBB.DDBB_FIBA_PSWD,
db = DDBB.DDBB_FIBA_NAME,
charset = DDBB.DDBB_FIBA_CHARSET,
cursorclass=pymysql.cursors.DictCursor)
with conn.cursor() as cursor:
print("id player: {}".format(p["id"]))
print("args: {}".format(args))
cursor.execute("select sc.* from tbl030_shots_chart sc, tbl006_player_team pt, tbl007_game g, tbl004_jornada j, tbl012_competition c where pt.id = %s and pt.id_player_feb = sc.id_fiba and sc.id_game = g.id and g.id_jornada = j.id and j.id_competition = c.id and c.id = %s", args)
data = cursor.fetchall()
print("data: {}".format(data))
print("Total rows: {}".format(cursor.rowcount))
if cursor.rowcount > 0:
for s in data:
x = float(FIBASCReport.adjust_x(s["x"]))
y = float(FIBASCReport.adjust_y(s["y"]))
color = tcm.image.getpixel((x,y))
color = ("#%02x%02x%02x" % color).upper()
if tcm.exists_color(color):
if int(s["m"]) == 0:
tcm.set_scored_shots(color, 1)
else:
tcm.set_failed_shots(color, 1)
else:
if int(s["m"]) == 0:
tcm.set_scored_shots("OTROS", 1)
else:
tcm.set_failed_shots("OTROS", 1)
else:
#tcm = None
print("Jugadora con id: {} NO ha realizado ningún tiro en competición: {}".format(p["id"], id_competition))
return tcm
In this code, cursor.fetchall() returns data the first query but the next querys returns empty results.
How can I run several querys? I'm using mySQL 8.0 and Python 3.6
Its because you are using the same cursor each time. create a new instance of the cursor each time you loop through to excecute the query. After the first query is run the cursor is already positioned after all the data. Hence no rows returned after that
You can also try this:
Look at the documentation for MySQLCursor.execute().
It claims that you can pass in a multi parameter that allows you to run multiple queries in one string.
If multi is set to True, execute() is able to execute multiple statements specified in the operation string.
multi is an optional second parameter to the execute() call:
operation = 'SELECT 1; INSERT INTO t1 VALUES (); SELECT 2'
for result in cursor.execute(operation, multi=True):
My usecase is to write create a temp table in the postgres database and fetch records from it and insert into a different table.
The code i used is:
import psycopg2
import sys
import pprint
from __future__ import print_function
from os.path import join,dirname,abspath
import xlrd
import os.path
newlist = []
itemidlist = []
def main():
conn_string = "host='prod-dump.cvv9i14mrv4k.us-east-1.rds.amazonaws.com' dbname='ebdb' user='ebroot' password='*********'"
# print the connection string we will use to connect
# print "Connecting to database" % (conn_string)
# get a connection, if a connect cannot be made an exception will be raised here
conn = psycopg2.connect(conn_string)
# conn.cursor will return a cursor object, you can use this cursor to perform queries
cursor = conn.cursor()
dealer_id = input("Please enter dealer_id: ")
group_id = input("Please enter group_id: ")
scriptpath = os.path.dirname('__file__')
filename = os.path.join(scriptpath, 'Winco - Gusti.xlsx')
xl_workbook = xlrd.open_workbook(filename, "rb")
xl_sheet = xl_workbook.sheet_by_index(0)
print('Sheet Name: %s' % xl_sheet.name)
row=xl_sheet.row(0)
from xlrd.sheet import ctype_text
print('(Column #) type:value')
for idx, cell_obj in enumerate(row):
cell_type_str = ctype_text.get(cell_obj.ctype, 'unknown type')
#print('(%s) %s %s' % (idx, cell_type_str, cell_obj.value))
num_cols = xl_sheet.ncols
for row_idx in range(0, xl_sheet.nrows): # Iterate through rows
num_cols = xl_sheet.ncols
id_obj = xl_sheet.cell(row_idx, 1) # Get cell object by row, col
itemid = id_obj.value
#if itemid not in itemidlist:
itemidlist.append(itemid)
# execute our Query
'''
cursor.execute("""
if not exists(SELECT 1 FROM model_enable AS c WHERE c.name = %s);
BEGIN;
INSERT INTO model_enable (name) VALUES (%s)
END;
""" %(itemid,itemid))
'''
cursor.execute("drop table temp_mbp1")
try:
cursor.execute("SELECT p.model_no, pc.id as PCid, g.id AS GROUPid into public.temp_mbp1 FROM products p, \
model_enable me, products_clients pc, groups g WHERE p.model_no = me.name \
and p.id = pc.product_id and pc.client_id = %s and pc.client_id = g.client_id and g.id = %s"\
% (dealer_id,group_id)
except (Exception, psycopg2.DatabaseError) as error:
print(error)
cursor.execute("select count(*) from public.temp_mbp1")
# retrieve the records from the database
records = cursor.fetchall()
# print out the records using pretty print
# note that the NAMES of the columns are not shown, instead just indexes.
# for most people this isn't very useful so we'll show you how to return
# columns as a dictionary (hash) in the next example.
pprint.pprint(records)
if __name__ == "__main__":
main()
The try except block in between the program is not throwing any error but the table is not getting created in the postgres database as i see in the data admin.
The output shown is:
Please enter dealer_id: 90
Please enter group_id: 13
Sheet Name: Winco Full 8_15_17
(Column #) type:value
[(3263,)]
Thanks,
Santosh
You didn't commit the changes, so they aren't saved in the database. Add to the bottom, just below the pprint statement:
conn.commit()