I'm migrating a script from another language to Python. I watered this down on the specifics of the database calls etc... but this is what the file looks like. I intentionally made some queries fail as I was testing the transaction and it did not rollback() the queries executed prior to the forced error. I am a little confused as how to the transactions work with Python, the example I followed was this one, it was a loop with several queries nested within transactions so I adapted the code according to what I understood from it.
#!/usr/bin/python
import MySQLdb
import thread
import os
# Open database connection
# added local_infile=1 to allow the import to work, otherwise you get an error
db = MySQLdb.connect(CONNECTION ARGS...)
# define our function that will be called from our thread
def import_queued_file(conn,distid):
# prepare a cursor object using cursor() method
cursor = conn.cursor()
# total lines imported for all files for a distributor
total_lines_imported = 0
# current lines imported for each file on each iteration
current_lines_imported = 0
# default this to 0, this will have the total lines for our imports on each iteration
previous_lines_imported = 0
# initialize the file exists flag to 0
file_exists = 0
# sql statement to retrieve the file(s) for a specific distributor
sql = """
SELECT
...
FROM ...
WHERE ...
"""
# execute the sql statement
cursor.execute(sql)
# if we have records, execute the code below
if (cursor.rowcount > 0):
# set the records to the files variable
files = cursor.fetchall()
# set a variable to count iterations
# we'll use this to determine if we need to drop the table
cnt = 0
# keep track of the total number of lines imported per distributor (may be multiple files)
lines_imported = 0
# loop the recordset
for col in files:
# increment the cnt variable
cnt += 1
# set file_exists to 0 at the beginning of the iteration
file_exists = 0
# set some variables to be used in our sql load data statement
var1 = col[1]
var2 = col[2]
....
# this is the path of our file that we will be using for MySQL LOAD DATA also
# TODO: REFACTOR SO THAT THE /home/webex/backup/ IS NOT HARD CODED
inventoryfile = "/path/to/file/%s" % (filepath)
# check to see if we have a file
if (os.path.exists(inventoryfile)):
try:
# set file exists to true
file_exists = 1
# if cnt > 1, it means we have more than 1 file for this distributor
# only drop the table if this is the first iteration
if (cnt == 1):
# drop table sql statement
sql = "DROP TABLE IF EXISTS %s" % (temptable)
# execute the sql command
cur = conn.cursor()
cur.execute(sql)
cur.close()
# assign the create table statement to the sql variable
sql = """
CREATE TABLE IF NOT EXISTS
.......
.......
) ENGINE=MyISAM DEFAULT CHARSET=utf8
""" % (temptable)
# execute the sql statement
cur = conn.cursor()
cur.execute(sql)
cur.close()
# query the temptable to see if we have any records
sql = "SELECT COUNT(0) AS total FROM %s" % (temptable)
cur = conn.cursor()
cur.execute(sql)
cur.close()
# get the count of how many records exist in the database
number_of_line_items = cur.fetchall()
previous_lines_imported = number_of_line_items[0][0]
# load data local infile sql statement
sql = """
LOAD DATA LOCAL INFILE ...
"""
# execute the load data infile sql statement
cur = conn.cursor()
cur.execute(sql)
cur.close()
# clean up the table by removing...
# rows that don't have a part_number,
# rows that have part_number's less than 3 characters
sql = """
DELETE FROM ...
""" % (temptable)
# execute the delete query
cur = conn.cursor()
cur.execute(sql)
cur.close()
# query the temptable to see if we have any records after the import
sql = "SELECT COUNT(0) AS total FROM %s" % (temptable)
# execute the count query
cur = conn.cursor()
cur.execute(sql)
cur.close()
# get the count of how many records exist in the database after the import
number_of_line_items = cur.fetchall()
# get the current lines imported
current_lines_imported = number_of_line_items[0][0] - previous_lines_imported
# add the current lines imported to the total lines imported
total_lines_imported += current_lines_imported
# update distributor_file_settings table last_updated_on field
sql = """
UPDATE ...
""" % (file_id,distributor__id)
print sql
# execute the update query
cur = conn.cursor()
cur.execute(sql)
cur.close()
# close cursor
conn.commit()
except:
conn.rollback()
# no records exists for this distributor
else:
print "dist doesn't exist"
cursor.close()
import_queued_file(db,42)
# prepare a cursor object using cursor() method
cursor = db.cursor()
# select distinct file settings
sql = """
SELECT ...
"""
# disconnect from server
db.close()
After reviewing the code again and again, the issue happened to be the table type. After changing it to INNODB it worked as expected.
Related
I wrote a fairly simple SQL while loop and tried to submit it via pyodbc cursor. But it didn't work, while working perfectly fine in SQL Server Management Studio.
My understanding is that one cannot pass more than one statement with the cursor. But then how does one execute a SQL while loop? I know I can do the below query with the while loop inside the python by cursor.rowcount, but my question is about generic queries with various SQL functions (like while here).
conn = get_output_conn(env=ENVIRONMENT)
conn.autocommit=True
cursor = conn.cursor()
query = """WHILE 1 = 1
BEGIN
BEGIN TRANSACTION;
DELETE TOP(2000)
FROM table with(holdlock)
WHERE ReportDate = '2020-08-23';
IF ##ROWCOUNT < 1 BREAK;
COMMIT TRANSACTION;
END"""
cursor.execute(query)
cursor.commit()
Try testing your rowcount condition after the commit transaction; statement. The following works for me...
import pyodbc
conn = pyodbc.connect(
autoCommit=False,
driver="/usr/local/lib/libtdsodbc.so",
tds_version="7.4",
database="StackOverflow",
port=...,
server="...",
user="...",
password="..."
)
query1 = """drop table if exists dbo.DeleteExample;"""
cursor1 = conn.cursor()
cursor1.execute(query1)
cursor1.commit()
cursor1.close()
query2 = """
select cast('2020-08-23' as date) as ReportDate
into dbo.DeleteExample
from sys.objects a, sys.objects b"""
cursor2 = conn.cursor()
cursor2.execute(query2)
# About 10,000 rows depending on your database
print(cursor2.rowcount, "rows inserted")
cursor2.commit()
cursor2.close()
query3 = """
declare #RowCount int;
while 1=1
begin
begin transaction t1;
delete top (2000)
from dbo.DeleteExample
where ReportDate = '2020-08-23';
set #RowCount = ##RowCount;
commit transaction t1;
if #RowCount < 1 break;
end"""
cursor3 = conn.cursor()
cursor3.execute(query3)
# "2000" which only is the first rowcount...
print(cursor3.rowcount, "rows deleted")
cursor3.commit()
cursor3.close()
Which outputs...
% python ./example.py
(10609, 'rows inserted')
(2000, 'rows deleted')
Executing select count(1) from StackOverflow.dbo.DeleteExample in SSMS returns a count of 0.
cmd_connection1 = ("mysql -uuser -ppw -h127.0.0.1 mydatabase -s"
" -N -e \'select * from mytable where ID=\""+ID+"\";\'")
Using Python 2.4, I want to check if there is any result(as a row)
I used :
if not line1 :
...
if line1 == "Empty" :
...
But there is no result.
A working example I use every day.
Im using mysql connector:
You can download from:
https://dev.mysql.com/downloads/connector/python/
import mysql.connector
# impor mysql connector
cnx = mysql.connector.connect(user='your_user', password='your_pwd', host='you_host',database='your_database')
#create a conecction to mysql server
# change user, password, host and database according your needs
id_a_buscar =15
# i will search this value in my database
cursor = cnx.cursor()
# create a cursor
cursor.execute("""SELECT
titulo,
clave
FROM
historico
WHERE
libro_pk =%s""", (id_a_buscar,))
# execute a query
# %s stores a variable
# id_a_buscar is assigned to s
# so the REAL query is SELECT titulo, clave FROM historico WHERE libro_pk = 15
resultados = cursor.fetchall()
# store query results in resultados
count = cursor.rowcount
# count how many rows return after the query
if count > 0:
# if there are records
else:
# if there are NO records
You should use python 3
I have some Python code the selects data from Oracle spatial and inserts into Spatialite. My problem is that the cursor contains the geometry in binary and I can’t figure out how to read the binary into the Spatialite insert statement. Just to added this all works if I use WKT but some of the geometries are too long hence the reason for the binary format.
Can anyone help please?
# Import system modules
import cx_Oracle
from pyspatialite import dbapi2 as sl_db
def db_connect():
# Build connect from TNS names
o_db = cx_Oracle.connect("xxxxx", "xxxxx", "xxxxx_gl_dev")
cursor = o_db.cursor()
return cursor
def db_lookup(cursor):
# Select records
sql = "SELECT sdo_util.to_wkbgeometry(a.shape), a.objectid FROM span a WHERE a.objectid = 1382372"
cursor.execute(sql)
row = cursor.fetchall()
return row
def db_insert(row):
# Insert Rows in new spatailite table
database_name = 'C:\\Temp\\MYDATABASE.sqlite'
db_connection = sl_db.connect(database_name)
db_cursor = db_connection.cursor()
sql = 'INSERT INTO "SPAN_OFL" ("geometry", "OBJECTID") Values GeomFromWKB(?,27700),?);'
db_cursor.executemany(sql, row)
db_connection.commit()
db_connection.close()
# main code
cursor = db_connect()
row = db_lookup(cursor)
db_insert(row)
I'm attempting to get a python script to insert data into a database without having it drop the table first.. I'm sure this isn't hard to do but I can't seem to get the code right..
Here is the full python script..
#!/usr/bin/python
# -*- coding: utf-8 -*-
import requests
import hashlib
import time
import MySQLdb
#Dont forget to fill in PASSWORD and URL TO saveTemp (twice) in this file
sensorids = ["28-000004944b63", "28-000004c01b2c"]
avgtemperatures = []
for sensor in range(len(sensorids)):
temperatures = []
for polltime in range(0,3):
text = '';
while text.split("\n")[0].find("YES") == -1:
# Open the file that we viewed earlier so that python can see what is in it. Replace the serial number as before.
tfile = open("/sys/bus/w1/devices/"+ sensorids[sensor] +"/w1_slave")
# Read all of the text in the file.
text = tfile.read()
# Close the file now that the text has been read.
tfile.close()
time.sleep(1)
# Split the text with new lines (\n) and select the second line.
secondline = text.split("\n")[1]
# Split the line into words, referring to the spaces, and select the 10th word (counting from 0).
temperaturedata = secondline.split(" ")[9]
# The first two characters are "t=", so get rid of those and convert the temperature from a string to a number.
temperature = float(temperaturedata[2:])
# Put the decimal point in the right place and display it.
temperatures.append(temperature / 1000 * 9.0 / 5.0 + 32.0)
avgtemperatures.append(sum(temperatures) / float(len(temperatures)))
print avgtemperatures[0]
print avgtemperatures[1]
#connect to db
db = MySQLdb.connect("localhost","user","password","temps" )
#setup cursor
cursor = db.cursor()
#create temps table
cursor.execute("DROP TABLE IF EXISTS temps")
sql = """CREATE TABLE temps (
temp1 FLOAT,
temp2 FLOAT )"""
cursor.execute(sql)
#insert to table
try:
cursor.execute("""INSERT INTO temps VALUES (%s,%s)""",(avgtemperatures[0],avgtemperatures[1]))
db.commit()
except:
db.rollback()
#show table
cursor.execute("""SELECT * FROM temps;""")
print cursor.fetchall()
((188L, 90L),)
db.close()
This is the part I need assistance with..
If I have it drop the table it works fine but I don't want it to drop the table, just insert the new data into the same table.
#connect to db
db = MySQLdb.connect("localhost","user","pasword1","temps" )
#setup cursor
cursor = db.cursor()
#create temps table
cursor.execute("DROP TABLE IF EXISTS temps")
sql = """CREATE TABLE temps (
temp1 FLOAT,
temp2 FLOAT )"""
cursor.execute(sql)
#insert to table
try:
cursor.execute("""INSERT INTO temps VALUES (%s,%s)""",(avgtemperatures[0],avgtemperatures[1]))
db.commit()
except:
db.rollback()
#show table
cursor.execute("""SELECT * FROM temps;""")
print cursor.fetchall()
((188L, 90L),)
db.close()
You shouldn`t have to drop a table each time you want to enter data. In fact, it defeats the whole purpose of the database since you will remove all the previous data each time you run your script.
You should ask to create the table but only if it does not exists. Use the following.
sql = """CREATE TABLE IF NOT EXISTS temps (
temp1 FLOAT,
temp2 FLOAT )"""
cursor.execute(sql)
I've had this problem with updating. Try adding COMMIT to the end of your sql. I use psycopg2 to connect to a postgresql database. Here is an example.
def simple_insert():
sql = '''INSERT INTO films VALUES ('UA502', 'Bananas', 105, '1971-07-13', 'Comedy', '82 minutes'); COMMIT;'''
try:
conn = psycopg2.connect(database)
cur = conn.cursor()
cur.execute(sql)
except:
raise
I think your problem is your not saving the transaction and the COMMIT command should fix it.
I tried to fill a table in a database using MySQLdb. It did not give any errors, and once gave the warning
main.py:23: Warning: Data truncated for column 'other_id' at row 1
cur.execute("INSERT INTO map VALUES(%s,%s)",(str(info[0]).replace('\n',''), str(info[2].replace('\n','').replace("'",""))))
so I thought it was working fine. However, when it was finished and I did a row count it turned out that nothing was added. Why was the data not added to the database? The code is below
def fillDatabase():
db = MySQLdb.connect(host="127.0.0.1",
user="root",
passwd="",
db="uniprot_map")
cur = db.cursor()
conversion_file = open('idmapping.dat')
for line in conversion_file:
info = line.split('\t')
cur.execute("INSERT INTO map VALUES(%s,%s)",(str(info[0]).replace('\n',''), str(info[2].replace('\n','').replace("'",""))))
def test():
db = MySQLdb.connect(host="127.0.0.1",
user="root",
passwd="",
db="uniprot_map")
cur = db.cursor()
cur.execute("SELECT COUNT(*) FROM map")
rows = cur.fetchall()
for row in rows:
print row
def main():
fillDatabase()
test()
You need to do a db.commit() after adding all of the entries. Even if the update is not transactional, the DBAPI imposes an implicit transaction on every change.