I'm currently trying to speed up the process of piping our data through from 3rd party tools. I've written a function to push through insert statements to our MySQL database in bulk, but am getting the following error:
_mysql_exceptions.ProgrammingError: (2014, "Commands out of sync; you can't run this command now")
The function I've written is as follows, taking in record_array (an array of dicts containing the data to be pushed through to the MySQL table) and tablename (a string containing the name of the table):
def insert_update_bulk(self, record_array, tablename):
session = self.Session()
self.merge_query = ""
for r in record_array:
query = '''INSERT INTO {} ('''.format(tablename)
for k in r.keys():
query += '{}, '.format(k)
query = query[:-2] + ''') VALUES ('''
for v in r.values():
if type(v) == int:
query += '{},'.format(v)
else:
query += '"{}",'.format(str(v).replace('"', "'").replace('\n', ' ').replace('\r', ' '))
query = query[:-1] + ''') ON DUPLICATE KEY UPDATE '''
for k in r.keys():
if type(r[k]) == int:
query += '{}={},'.format(k, str(r[k]).replace('"', "'")).replace("'None'", 'NULL')
else:
query += '{}="{}",'.format(k, str(r[k]).replace('"', "'")).replace("'None'", 'NULL')
query = query[:-1] + '; '
self.merge_query += query
session.execute(self.merge_query.replace('%', '%%'))
self.Session.close_all()
Related
I am trying to populate an azure sql database using pyodbc.
I have around 14000item to push into the database per months, and running and executing each single upsert on it is really long.
I am so trying to run them in batch and push them all at once. Sadly, when doing this I am not getting anything in the database. Is there any way to make this works ?
i = 0
queryUpdate = ""
for values in valuesList:
if i == 100:
i = 0
queryUpdate = queryUpdate.replace('\'NULL\'', "NULL") #set null to real sql null
queryUpdate = queryUpdate.replace("True", "1") #replace true with 1 to match the bit type
queryUpdate = queryUpdate.replace("False", "0") #replace false with 0 to match the bit type
try:
cursor.execute(queryUpdate)
except pyodbc.Error as ex:
print(f"[x] Error when running the query:\n[x] {queryUpdate}\n[x] Exception: {ex}")
queryUpdate = ""
queryUpdate += f"""\n
if exists (select * from {tableName} with (updlock,serializable) where {colNames[keyIndex]} = {values[keyIndex]})
begin
update {tableName} set """
for i in range(len(colNames)):
queryUpdate += f"{colNames[i]} = '{values[i]}'"
if i != len(colNames) - 1:
queryUpdate += ","
queryUpdate += f"""
where {colNames[keyIndex]} = {values[keyIndex]}
end
else
begin
insert into {tableName} ({','.join(colNames)})
values {tuple(values)}
end;
"""
i+=1
try:
conn.commit()
except pyodbc.Error as ex:
print(f"[x] Error when commiting to the database.\n[x] Exception: {ex}")
else:
print("[+] Commit confirmed !")
cursor.close()
conn.close()
print("[+] Connection closed.")
i have two function (in python). The first function defines a new variable which i have to insert in a sql table (first column). The second one, does the same thing, but i want to insert its variable (the second one) near the first variable, so in the second column but in the same line. How can i do with sql?.
connloc = sqlite3.connect("request.db")
sqlloc = "create table requests (" \
" chat_id INTEGER NOT NULL PRIMARY KEY,"\
" locpar varchar(20)," \
" stoppar varchar(20)," \
" locdes varchar(20) ," \
" stopdes varchar(20) );"
connloc.execute(sqlloc)
def name_loc(chat, message):
for i in result:
if message.text == i:
item = [i]
cloc = connloc.cursor()
cloc.execute("INSERT INTO requests(locpar) VALUES (?);", item)
connloc.commit()
def name_stop(chat, message):
for i in result:
for t in result[i]:
if message.text == t:
item = [t]
cloc = connloc.cursor()
cloc.execute("INSERT INTO requests(stoppar) VALUES (?);", item)
connloc.commit()
I would break it up into a two step process by defining two methods, one for table generation and then another second method for populating the new table like this:
def create_table(ptbl):
""" Assemble DDL (Data Definition Language) Table Create statement and build
sqlite3 db table
Args:
string: new db table name.
Returns:
Status string, '' or 'SUCCESS'.
"""
retval = ''
sqlCmd = ''
try:
conn = sqlite3.connect(sqlite_file)
c = conn.cursor()
if ptbl == 'TBL_EXAMPLE':
sqlCmd = 'CREATE TABLE IF NOT EXISTS ' + ptbl + ' (FIELD1 TEXT, FIELD2 INTEGER, FIELD3 TEXT, ' \
'FIELD4 TEXT, FIELD5 TEXT)'
else:
pass
if sqlCmd != '':
c.execute(sqlCmd)
conn.commit()
conn.close()
retval = 'SUCCESS'
except Error as e:
retval = 'FAIL'
print(e)
return retval
and then populate it as you like with the values (inserting your new row with those two specific values you mentioned).
Now, I'm populating from a csv file here, but I thinkit'll give you a really good solid start on this task.
def populate_tbl_file_marker_linenums(p_fml_tbl, p_fml_datafile):
""" Read csv and load data into TBL_FILE_MARKER_LINENUMS table ...
Args:
p_fml_tbl (TEXT) target table name
p_fml_datafile (TEXT) name of csv file to load into table
Returns:
retval (TEXT) - Status of method, e.g., 'SUCCESS'
"""
retval = ''
mode = 'r'
try:
conn = sqlite3.connect(sqlite_file)
c = conn.cursor()
csv_dataset = open(p_fml_datafile, mode)
csv_reader = csv.reader(csv_dataset)
c.executemany('INSERT INTO ' + p_fml_tbl + ' (FIELD1, FIELD2, FIELD3, FIELD4, FIELD5) VALUES (?, ?, ?, ?, ?)', csv_reader)
conn.commit()
conn.close()
retval = 'SUCCESS'
except Error as e:
print(e)
return retval
I'm trying to execute the same query but with different data but I always get data the first time. The others times, dispite of there are data for the querys in the data base, mysql returns empty data.
This is the code:
def get_team_colour_map(self, players, id_competition):
tcm = FIBAColourMap()
for p in players:
args = [p["id"], id_competition]
conn = pymysql.Connect(host = DDBB.DDBB_FIBA_HOST,
user = DDBB.DDBB_FIBA_USER,
password = DDBB.DDBB_FIBA_PSWD,
db = DDBB.DDBB_FIBA_NAME,
charset = DDBB.DDBB_FIBA_CHARSET,
cursorclass=pymysql.cursors.DictCursor)
with conn.cursor() as cursor:
print("id player: {}".format(p["id"]))
print("args: {}".format(args))
cursor.execute("select sc.* from tbl030_shots_chart sc, tbl006_player_team pt, tbl007_game g, tbl004_jornada j, tbl012_competition c where pt.id = %s and pt.id_player_feb = sc.id_fiba and sc.id_game = g.id and g.id_jornada = j.id and j.id_competition = c.id and c.id = %s", args)
data = cursor.fetchall()
print("data: {}".format(data))
print("Total rows: {}".format(cursor.rowcount))
if cursor.rowcount > 0:
for s in data:
x = float(FIBASCReport.adjust_x(s["x"]))
y = float(FIBASCReport.adjust_y(s["y"]))
color = tcm.image.getpixel((x,y))
color = ("#%02x%02x%02x" % color).upper()
if tcm.exists_color(color):
if int(s["m"]) == 0:
tcm.set_scored_shots(color, 1)
else:
tcm.set_failed_shots(color, 1)
else:
if int(s["m"]) == 0:
tcm.set_scored_shots("OTROS", 1)
else:
tcm.set_failed_shots("OTROS", 1)
else:
#tcm = None
print("Jugadora con id: {} NO ha realizado ningún tiro en competición: {}".format(p["id"], id_competition))
return tcm
In this code, cursor.fetchall() returns data the first query but the next querys returns empty results.
How can I run several querys? I'm using mySQL 8.0 and Python 3.6
Its because you are using the same cursor each time. create a new instance of the cursor each time you loop through to excecute the query. After the first query is run the cursor is already positioned after all the data. Hence no rows returned after that
You can also try this:
Look at the documentation for MySQLCursor.execute().
It claims that you can pass in a multi parameter that allows you to run multiple queries in one string.
If multi is set to True, execute() is able to execute multiple statements specified in the operation string.
multi is an optional second parameter to the execute() call:
operation = 'SELECT 1; INSERT INTO t1 VALUES (); SELECT 2'
for result in cursor.execute(operation, multi=True):
I'm new to using sqlite3 with python. I'm having trouble with making the insert_row function in my database class. From the manual on the execute function it appears that I'm doing everything correctly https://docs.python.org/2/library/sqlite3.html. This function does return a number which appears to be correct for lastrowid. However, when I access sqlite3 via command line the table is empty.
def insert_row(self, table, input_dict):
'''
(str, dict) -> int
Add a row to a table in the database
'''
count = 0
col_names = input_dict.keys()
insert_sql = 'INSERT INTO ' + self.sanitize(table) + ' (' + utils.list_to_char_seperated(col_names, ', ', False) + ') VALUES ('
for col in col_names:
count += 1
insert_sql += "'" + self.sanitize(input_dict[col]) + "'"
if count < len(col_names):
insert_sql += ', '
else:
insert_sql += ');'
cur = self.conn.cursor()
cur.execute(insert_sql)
return cur.lastrowid
Accessing via command line.
$ sqlite3
SQLite version 3.11.0 2016-02-15 17:29:24
Enter ".help" for usage hints.
Connected to a transient in-memory database.
Use ".open FILENAME" to reopen on a persistent database.
sqlite> .open blood_cultures.db
sqlite> .tables
vein_draws
sqlite> SELECT * FROM vein_draws;
sqlite>
Also if I take the result of the insert_sql variable and insert it directly into the database via a command line connection this works. So I do not believe it is my insert statement.
Any ideas on what I am doing wrong here?
This is my first post here and I found so many answers here I am really confident, that some can help me. Because I work with Python for only about half a year now, it maybe is a very stupid beginners question. Forgive me so far...
In my project I have several different functions. Two of them are listed below. mysqlLogbookIndex is a thread witch show refresh a list with the names of some table from a database. The connection to the DB is existing at the moment, the function is called and so far it works fine. At the end I can see a tuple containing all the table names.
But now the second function, named create_flight comes in. It is a callback for a Tkinter button and creates some new tables in my database. It uses the same preopened connection than mysqlLogbookIndex.
I expected to see the new tables is my tuple in the next cycle of mysqlLogbookIndex but what happened is that result turned into a None.
Do you know why?
Widget function:
def create_flight(self):
# *********************************************************************************************
# * Create new flight table *
# *********************************************************************************************
if self.sql_write.get():
# ********************************************
# * Try to connect message *
# ********************************************
self.printLog(self.lbl_sql_write, self.LANG['tryCreateTable'], 'normal')
logging.info('Creating table for flight tracking...')
# ********************************************
# * Create name tables *
# ********************************************
event = self.oprSett['mysql']['event']
now = str(int(time.time()))
mainName = "tbl_%s_%s_" % (event, now)
trackingTable = mainName + "flighttrack"
logging.debug('Name of tracking table: %s', trackingTable)
unitsTable = mainName + "units"
logging.debug('Name of units table: %s', unitsTable)
headerTable = mainName + "header"
logging.debug('Name of header table: %s', headerTable)
# ********************************************
# * Read SQL parameter *
# ********************************************
logging.debug('Reading CSV file for table structure...')
csvFile = "config/newFlight.csv"
try:
sqlCsv = csv.reader(open(csvFile, 'rb'),
delimiter = ',',
quotechar = '"',
quoting = csv.QUOTE_ALL
)
except:
msg = csvFile
msg += "\n\n"
msg += self.LANG['e13']
tkMessageBox.showerror("Error 13", msg)
self.printLog(self.lbl_sql_write, self.LANG['e13'], 'error')
logging.error('File not found!')
#print "[Error 13] " + self.LANG['e13']
return 0
# Transfer data from CSV file into own list
sqlVars = []
for row in sqlCsv:
if len(row) == 4 and row[0][0] != "#": # No comment
sqlVars.append(row)
# *************************************************
# * Create SQL statement to create tracking table *
# *************************************************
# Head for creating new table
sql = "CREATE TABLE IF NOT EXISTS `%s` (\n" % trackingTable
sql += " `ID` int(11) NOT NULL AUTO_INCREMENT,\n" # Becomes primary key
# Parse SQL variables from CSV file
for row in sqlVars:
if len(row[2]) > 0: # Data type requires length
sql += " `%s` %s(%s) NOT NULL COMMENT '%s',\n" % (row[0], row[1], row[2], row[3])
else: # Data type not requires length
sql += " `%s` %s NOT NULL COMMENT '%s',\n" % (row[0], row[1], row[3])
# Footer of SQL statement for creating new table
sql += " PRIMARY KEY (`ID`)\n"
sql += ") ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_german1_ci AUTO_INCREMENT=0;\n"
sql += "\n"
# In debug mode print SQL statement to console
#logging.debug('SQL statement to create tracking table:\n%s', sql)
# **********************************************
# * Create SQL statement to create units table *
# **********************************************
# Head for creating new table
sql += "CREATE TABLE IF NOT EXISTS `%s` (\n" % unitsTable
sql += " `ID` int(11) NOT NULL AUTO_INCREMENT,\n" # Becomes primary key
sql += " `Dataref` varchar(10) COLLATE latin1_german1_ci NOT NULL,\n"
sql += " `Unit` varchar(10) COLLATE latin1_german1_ci NOT NULL,\n"
sql += " PRIMARY KEY (`ID`)\n"
sql += ") ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_german1_ci AUTO_INCREMENT=1 ;\n"
sql += "\n"
# Parse SQL variables from CSV file
for row in sqlVars:
# Insert units in tbl_units
sql += "INSERT INTO %s VALUES ('', '%s', '%s');\n" % (unitsTable, row[0], row[3])
sql += "\n"
# In debug mode print SQL statement to console
#logging.debug('SQL statement to create units table:\n%s', sql)
# ***********************************************
# * Create SQL statement to create header table *
# ***********************************************
# Head for creating new table
sql += "CREATE TABLE IF NOT EXISTS `%s` (\n" % headerTable
#sql += " `ID` int(11) NOT NULL AUTO_INCREMENT,\n" # Becomes primary key
sql += " `Parameter` char(21) COLLATE latin1_german1_ci NOT NULL,\n"
sql += " `Value` varchar(100) COLLATE latin1_german1_ci NOT NULL,\n"
sql += " PRIMARY KEY (`Parameter`)\n"
sql += ") ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_german1_ci AUTO_INCREMENT=1 ;\n"
sql += "\n"
# IGC syntax from: http://carrier.csi.cam.ac.uk/forsterlewis/soaring/igc_file_format/igc_format_2008.html
# Adding header parameters, some values are coming later
sql += "INSERT INTO %s VALUES ('AXXX001', '');\n" % (headerTable) # Manufacturer code
sql += "INSERT INTO %s VALUES ('HFFXA', '035');\n" % (headerTable) # Fix accuracy
sql += "INSERT INTO %s VALUES ('HFDTE', '');\n" % (headerTable) # UTC date of flight
sql += "INSERT INTO %s VALUES ('HFPLTPILOT', '');\n" % (headerTable) # Pilots name
sql += "INSERT INTO %s VALUES ('HFGTYGLIDERTYPE', 'KA8B');\n" % (headerTable) # Glider type
sql += "INSERT INTO %s VALUES ('HFGIDGLIDERID', 'D1389');\n" % (headerTable) # Glider callsign
sql += "INSERT INTO %s VALUES ('HFDTM100DATUM', 'WGS-1984');\n" % (headerTable) # GPS datum
sql += "INSERT INTO %s VALUES ('HFGPSGPS', 'X-PLANE 10');\n" % (headerTable) # Manufacturer of GPS module
sql += "INSERT INTO %s VALUES ('HFFTYFRTYPE', 'FLORIANMEISSNER,HCM');\n" % (headerTable) # Logger type
sql += "INSERT INTO %s VALUES ('HFRFWFIRMWAREVERSION', '%s');\n" % (headerTable, self.VERSION) # Firmware version
sql += "INSERT INTO %s VALUES ('HFRHWHARDWAREVERSION', '%s');\n" % (headerTable, self.XPLANEVERSION) # Hardware version
sql += "INSERT INTO %s VALUES ('HFCCLCOMPETITIONCLASS', 'CLUB');\n" % (headerTable) # Competition class
# ********************************************
# * Handover SQL statement to create table *
# * to DB. *
# ********************************************
logging.debug('SQL statement to create all tables:\n%s', sql)
try:
cur = self.conn.cursor()
cur.execute(sql)
cur.close()
except pymysql.Error, e:
tkMessageBox.showerror("Error 9", self.LANG['e9'])
self.printLog(self.lbl_sql_write, self.LANG['e9'], 'error')
#print "[Error 9] " + self.LANG['e9']
logging.error('Could not create tracking table!')
logging.debug(e)
return 0
# ********************************************
# * Print success message *
# ********************************************
logging.info('Tracking table created.')
self.printLog(self.lbl_sql_write, self.LANG['doneCreateTable'], 'success')
text = self.LANG['flightId'] + " " + trackingTable
self.printLog(self.lbl_sql_write, text, 'normal', timestamp=False)
# Make trackingTable public
self.tableName = trackingTable
# Enable tambour register to write in database
self.tambourInMysql = True
Refresh Loop:
def mysqlLogbookIndex(self):
delay = float(self.oprSett['logbook']['refresh_delay']) / 1000
# ********************************************************************
# * Run only if MySQL connection from writeMysql.py is active *
# ********************************************************************
while self.sql_write.get():
# ********************************************
# * Query to get list with tables *
# ********************************************
logging.info('Querying list with tables from database')
dbName = self.oprSett['mysql']['db']
sql = "SELECT TABLE_NAME\n"
sql += "FROM `information_schema`.`TABLES`\n"
sql += "WHERE `TABLE_SCHEMA` LIKE '%s'" % dbName
try:
cur = self.conn.cursor()
cur.execute(sql)
result = cur.fetchall()
cur.close()
except pymysql.Error, e:
logging.warning('Could not take flights from database!')
logging.debug(e)
self.printLog(self.lbl_sql_write, self.LANG['e20'], 'error')
#continue
else:
logging.info('Logbook refreshed.')
print result
Thanks in advance for all helpful posts...
Found the error more or less. It seems to be a MySQL issue.
In my original idea I connected to DB "hcm" and tried to get some table informations from "information_schema" DB. If the "hcm" was changed the query returned None.
Solution:
I established a second independent MySQL connection in my function for the refresh loop and connected to "information_schema" directly.