Develop Reference

Sqlite - Cannot operate on a closed database

I am trying to insert a small set of rows into sqlite using python and getting an error "Cannot operate on a closed database"
This is my code snippet:
import sqlite3
from sqlite3 import Error
db_file = "/home/sosuser/mediaserver/camera.db"
def create_connection(db_file):
conn = None
try:
conn = sqlite3.connect(db_file)
print(sqlite3.version)
except Error as e:
print(e)
finally:
if conn:
conn.close()
return conn
def create_task(conn, task):
sql = ''' INSERT INTO camerainfo(id, cameraid, maplat, maplong, name)
VALUES(?,?,?,?,?) '''
cur = conn.cursor()
cur.execute(sql, task)
def prepare_data(conn):
for cam in range(len(camID)):
print(camID[cam])
task = (cam, camID[cam], '12.972442','77.580643','testCAM')
create_task(conn, task)
conn.commit()
conn.close()
conn = create_connection(db_file)
prepare_data(conn)
Get the following error -
Traceback (most recent call last):
File "dumpCamera1.py", line 92, in <module>
prepare_data(conn)
File "dumpCamera1.py", line 86, in prepare_data
create_task(conn, task)
File "dumpCamera1.py", line 79, in create_task
cur = conn.cursor()
sqlite3.ProgrammingError: Cannot operate on a closed database.
Not sure where is my connection being closed. Might have done something very silly but would appreciate for pointers?
Thanks.

The finally clause in the create_connection function closes the connection before it's returned.
It looks as if you are trying to create a kind of context manager for the connection, but an sqlite3 Connection is already a context manager, so this is unnecessary.
You can do
with sqlite3.connect(dbfile) as conn:
print(sqlite3.version)
prepare_data(conn)
The connection will be closed automatically on exiting the context manager. You can trap errors raised inside the context manager by wrapping it in a try / except block.

Is opening a new MySQL cursor for each query slow?

I've seen some answers around here that open a new MySQL cursor before each query, then close it.
Is that slow? Shouldn't I be recycling a cursor, by passing it in as a parameter?
I have a program that does an infinite loop, so eventually the connection will time out after the default 8 hours.
Edit:
As requested, this is the relevant code that handles the SQL query:
def fetch_data(query):
try:
cursor = db.Cursor()
cursor.execute(query)
return cursor.fetchall()
except OperationalError as e:
db = fetchDb()
db.autocommit(True)
print 'reconnecting and trying again...'
return fetch_data(query)

Of course, re-connecting a connection for thousands of times will take much more time. You'd better set it as a property of your class, like this:
class yourClass():
self.db = ...
self.cursor = self.con.Cursor()
# do something
def fetch_data(self, query):
try:
if self.cursor:
self.cursor.execute(query)
else:
raise OperationalError
return self.cursor.fetchall()
except OperationalError as e:
self.db = fetchDb()
self.db.autocommit(True)
print 'reconnecting and trying again...'
return fetch_data(query)

Sqlite3 with Python - Query from external file

I would like to create my DB by an external file like:
database = "../data/cm4payroll.db"
query = "../data/emdb.sql"
# Datenbankverbindung herstellen
self.connection = sqlite3.connect(self.database)
self.cursor = self.connection.cursor()
# Datenbank erstellen
self.cursor.execute(self.query)
Traceback:
self.cursor.execute(self.query)
sqlite3.OperationalError: near ".": syntax error

You need to read the file contents, and pass it to cursor.executescript() instead:
self.connection = sqlite3.connect(self.database)
self.cursor = self.connection.cursor()
with open(self.query) as queryfile:
self.cursor.executescript(queryfile.read())
Your error shows you were trying to execute the filename as a SQL statement; cursor.execute() can only handle actual SQL strings, not filenames.

Python: MySQLdb LOAD DATA INFILE silently fails

I am attempting to use a Python script to import a csv file into a MySQL database.
It seems to fail silently.
Here is my code:
#!/usr/bin/python
import MySQLdb
class DB:
host = 'localhost'
user = 'root'
password = '**************'
sqldb = 'agriculture'
conn = None
def connect(self):
self.conn = MySQLdb.connect(self.host,self.user,self.password,self.sqldb )
def query(self, sql, params=None):
try:
cursor = self.conn.cursor()
if params is not None:
cursor.execute(sql, params)
else:
cursor.execute(sql)
except (AttributeError, MySQLdb.OperationalError):
self.connect()
cursor = self.conn.cursor()
if params is not None:
cursor.execute(sql, params)
else:
cursor.execute(sql)
print vars(cursor)
return cursor
def load_data_infile(self, f, table, options=""):
sql="""LOAD DATA LOCAL INFILE '%s' INTO TABLE %s FIELDS TERMINATED BY ',';""" % (f,table)
self.query(sql)
db = DB()
pathToFile = "/home/ariggi/722140-93805-sqltest.csv"
table_name = "agriculture.degreedays"
db.load_data_infile(pathToFile, table_name)
In an attempt to debug this situation I am dumping the cursor object to the screen within the "query()" method. Here is the output:
{'_result': None, 'description': None, 'rownumber': 0, 'messages': [],
'_executed': "LOAD DATA LOCAL INFILE
'/home/ariggi/722140-93805-sqltest.csv' INTO TABLE degreedays FIELDS
TERMINATED BY ',';", 'errorhandler': >, 'rowcount': 500L, 'connection': , 'description_flags': None,
'arraysize': 1, '_info': 'Records: 500 Deleted: 0 Skipped: 0
Warnings: 0', 'lastrowid': 0L, '_last_executed': "LOAD DATA LOCAL
INFILE '/home/ariggi/722140-93805-sqltest.csv' INTO TABLE agriculture.degreedays
FIELDS TERMINATED BY ',';", '_warnings': 0, '_rows': ()}
If I take the "_last_executed" query, which is
LOAD DATA LOCAL INFILE '/home/ariggi/722140-93805-sqltest.csv' INTO TABLE agriculture.degreedays FIELDS TERMINATED BY ',';
and run it through the mysql console it works as expected and fills the table with rows. However when I execute this script my database table remains empty.
I am pretty stumped and could use some help.

Try calling db.conn.commit() at the end of your code to make the changes permanent. Python by default does not use the "autocommit" mode, so until you issue a commit the DB module regards your changes as part of an incomplete transaction.
As #AirThomas points out in a comment it helps to us a "context manager" - though I'd say the correct formulation was
with conn.cursor() as curs:
do_something_with(curs)
because this will automatically commit any changes unless the controlled code raises an exception.

How to enable MySQL client auto re-connect with MySQLdb?

I came across PHP way of doing the trick:
my_bool reconnect = 1;
mysql_options(&mysql, MYSQL_OPT_RECONNECT, &reconnect);
but no luck with MySQLdb (python-mysql).
Can anybody please give a clue? Thanks.

I solved this problem by creating a function that wraps the cursor.execute() method since that's what was throwing the MySQLdb.OperationalError exception. The other example above implies that it is the conn.cursor() method that throws this exception.
import MySQLdb
class DB:
conn = None
def connect(self):
self.conn = MySQLdb.connect()
def query(self, sql):
try:
cursor = self.conn.cursor()
cursor.execute(sql)
except (AttributeError, MySQLdb.OperationalError):
self.connect()
cursor = self.conn.cursor()
cursor.execute(sql)
return cursor
db = DB()
sql = "SELECT * FROM foo"
cur = db.query(sql)
# wait a long time for the Mysql connection to timeout
cur = db.query(sql)
# still works

I had problems with the proposed solution because it didn't catch the exception. I am not sure why.
I have solved the problem with the ping(True) statement which I think is neater:
import MySQLdb
con=MySQLdb.Connect()
con.ping(True)
cur=con.cursor()
Got it from here: http://www.neotitans.com/resources/python/mysql-python-connection-error-2006.html

If you are using ubuntu Linux there was a patch added to the python-mysql package that added the ability to set that same MYSQL_OPT_RECONNECT option (see here). I have not tried it though.
Unfortunately, the patch was later removed due to a conflict with autoconnect and transations (described here).
The comments from that page say:
1.2.2-7 Published in intrepid-release on 2008-06-19
python-mysqldb (1.2.2-7) unstable; urgency=low
[ Sandro Tosi ]
* debian/control
- list items lines in description starts with 2 space, to avoid reformat
on webpages (Closes: #480341)
[ Bernd Zeimetz ]
* debian/patches/02_reconnect.dpatch:
- Dropping patch:
Comment in Storm which explains the problem:
# Here is another sad story about bad transactional behavior. MySQL
# offers a feature to automatically reconnect dropped connections.
# What sounds like a dream, is actually a nightmare for anyone who
# is dealing with transactions. When a reconnection happens, the
# currently running transaction is transparently rolled back, and
# everything that was being done is lost, without notice. Not only
# that, but the connection may be put back in AUTOCOMMIT mode, even
# when that's not the default MySQLdb behavior. The MySQL developers
# quickly understood that this is a terrible idea, and removed the
# behavior in MySQL 5.0.3. Unfortunately, Debian and Ubuntu still
# have a patch right now which *reenables* that behavior by default
# even past version 5.0.3.

I needed a solution that works similarly to Garret's, but for cursor.execute(), as I want to let MySQLdb handle all escaping duties for me. The wrapper module ended up looking like this (usage below):
#!/usr/bin/env python
import MySQLdb
class DisconnectSafeCursor(object):
db = None
cursor = None
def __init__(self, db, cursor):
self.db = db
self.cursor = cursor
def close(self):
self.cursor.close()
def execute(self, *args, **kwargs):
try:
return self.cursor.execute(*args, **kwargs)
except MySQLdb.OperationalError:
self.db.reconnect()
self.cursor = self.db.cursor()
return self.cursor.execute(*args, **kwargs)
def fetchone(self):
return self.cursor.fetchone()
def fetchall(self):
return self.cursor.fetchall()
class DisconnectSafeConnection(object):
connect_args = None
connect_kwargs = None
conn = None
def __init__(self, *args, **kwargs):
self.connect_args = args
self.connect_kwargs = kwargs
self.reconnect()
def reconnect(self):
self.conn = MySQLdb.connect(*self.connect_args, **self.connect_kwargs)
def cursor(self, *args, **kwargs):
cur = self.conn.cursor(*args, **kwargs)
return DisconnectSafeCursor(self, cur)
def commit(self):
self.conn.commit()
def rollback(self):
self.conn.rollback()
disconnectSafeConnect = DisconnectSafeConnection
Using it is trivial, only the initial connect differs. Extend the classes with wrapper methods as per your MySQLdb needs.
import mydb
db = mydb.disconnectSafeConnect()
# ... use as a regular MySQLdb.connections.Connection object
cursor = db.cursor()
# no more "2006: MySQL server has gone away" exceptions now
cursor.execute("SELECT * FROM foo WHERE bar=%s", ("baz",))

you can separate the commit and the close for the connection...that's not cute but it does it.
class SqlManager(object):
"""
Class that handle the database operation
"""
def __init__(self,server, database, username, pswd):
self.server = server
self.dataBase = database
self.userID = username
self.password = pswd
def Close_Transation(self):
"""
Commit the SQL Query
"""
try:
self.conn.commit()
except Sql.Error, e:
print "-- reading SQL Error %d: %s" % (e.args[0], e.args[1])
def Close_db(self):
try:
self.conn.close()
except Sql.Error, e:
print "-- reading SQL Error %d: %s" % (e.args[0], e.args[1])
def __del__(self):
print "close connection with database.."
self.conn.close()

I had a similar problem with MySQL and Python, and the solution that worked for me was to upgrade MySQL to 5.0.27 (on Fedora Core 6; your system may work fine with a different version).
I tried a lot of other things, including patching the Python libraries, but upgrading the database was a lot easier and (I think) a better decision.

In addition to Liviu Chircu solution ... add the following method to DisconnectSafeCursor:
def __getattr__(self, name):
return getattr(self.cursor, name)
and the original cursor properties like "lastrowid" will keep working.

You other bet it to work around dropped connections yourself with code.
One way to do it would be the following:
import MySQLdb
class DB:
conn = None
def connect(self):
self.conn = MySQLdb.connect()
def cursor(self):
try:
return self.conn.cursor()
except (AttributeError, MySQLdb.OperationalError):
self.connect()
return self.conn.cursor()
db = DB()
cur = db.cursor()
# wait a long time for the Mysql connection to timeout
cur = db.cursor()
# still works