I have the following code:
import psycopg2
conn = psycopg2.connect(database="****", user="postgres", password="*****", host="localhost", port="5432")
print ("Opened database successfully")
cur = conn.cursor()
cur.execute('''select * from xyz''')
print ("Table created successfully")
conn.commit()
conn.close()
Like this i have some 50 -60 complex queries, but problem is some times the postgre sql Database throws the below error
Traceback (most recent call last):
File "C:/Users/ruw2/Desktop/SQL.py", line 87, in <module>
cur.execute('''select * from xyz;''')
psycopg2.DatabaseError: server conn crashed?
server closed the connection unexpectedly.
This probably means the server terminated abnormally before or while processing the request.
Looks like the connection is lost, how can i auto connect the Postgre database
Appreciate your help
I would rely on decorators - retry and reconnect:
import psycopg2
from tenacity import retry, wait_exponential, stop_after_attempt
from typing import Callable
def reconnect(f: Callable):
def wrapper(storage: DbStorage, *args, **kwargs):
if not storage.connected():
storage.connect()
try:
return f(storage, *args, **kwargs)
except psycopg2.Error:
storage.close()
raise
return wrapper
class DbStorage:
def __init__(self, conn: string):
self._conn: string = conn
self._connection = None
def connected(self) -> bool:
return self._connection and self._connection.closed == 0
def connect(self):
self.close()
self._connection = psycopg2.connect(self._conn)
def close(self):
if self.connected():
# noinspection PyBroadException
try:
self._connection.close()
except Exception:
pass
self._connection = None
#retry(stop=stop_after_attempt(3), wait=wait_exponential())
#reconnect
def get_data(self): # pass here required params to get data from DB
# ..
cur = self._connection.cursor()
cur.execute('''select * from xyz''')
# ..
Catch the exception and reconnect:
while True:
conn = psycopg2.connect(database="****", user="postgres", password="*****", host="localhost", port="5432")
cur = conn.cursor()
try:
cur.execute('''select * from xyz''')
except psycopg2.OperationalError:
continue
break;
Related
I'm writing a python code to read from mysql database:
def create_server_connection(host, user, password):
connection = None
try:
connection = pymysql.connect(host='localhost',
user='root',
password='pwd',
database='raw_data',
cursorclass=pymysql.cursors.DictCursor)
print("MySQL Database connection successful")
except err as error:
print(f"Error: '{error}'")
return connection
def read_query(connection, query):
cur = connection.cursor()
result = None
try:
cur.execute(query)
result = cur.fetchall()
return result
except err as error:
print(f"Error: '{error}'")
return cur
def get_Tables_byName(cursor, tableName):
q1 = f'''
SELECT table_name FROM raw_data.tables
where table_name like '{tableName}'; '''
res = []
cursor.execute(q1)
for row in cursor:
res.append(row[0])
return res
get_Tables_byName(cursor,'data_31942010201')
If I want to call get_Tables_byName function, what should I put in the first parameter? If I put cursor, the error message shows NameError: name 'cursor' is not defined
I currently have a long running python script (multiple days), which occasionally executes queries in a mysql db. The queries are executed from different threads.
I connect to the db as following:
import mysql.connector
import time
class DB():
connection = None
def __init__(self, host, user, password, database):
self.host = host;
self.user = user;
self.password = password;
self.database = database;
self.connect()
def connect(self):
try:
self.connection = mysql.connector.connect(
host = self.host,
user = self.user,
password = self.password,
database = self.database,
autocommit = True,
)
except Exception as exception:
print("[DBCONNECTION] {exception} Retrying in 5 seconds.".format(exception = str(exception)))
time.sleep(5)
self.connect()
db = DB("11.111.11.111", "mydb", "mydb", "mydb")
One of the functions to execute a query:
def getUsersDB(self):
db.connection.ping(reconnect=True)
cursor = db.connection.cursor()
cursor.execute("SELECT * FROM Users")
users = cursor.fetchall()
return users
I believe, that I'm far away from a good practice how I handle the connection. What is the correct way for such a case?
The problem with your approach is that db.connection.ping(reconnect=True) doesn't promise you a live connection.
You can read the function's description here:
https://dev.mysql.com/doc/connector-python/en/connector-python-api-mysqlconnection-ping.html
You can try to use this:
class DB:
connection = None
def __init__(
self,
host: str,
user: str,
password: str,
database: str
) -> None:
self.host = host
self.user = user
self.password = password
self.database = database
self.connection = self.init_conn()
def init_conn(
self,
):
return mysql.connector.connect(
host=self.host,
user=self.user,
password=self.password,
database=self.database,
)
def get_cursor(
self,
):
# This will try to reconnect also.
# In case it fails, you will have to create a new connection
try:
self.connection.ping(
reconnect=True,
attempts=3,
delay=5,
)
except mysql.connector.InterfaceError as err:
self.connection = self.init_conn()
return self.connection.cursor()
And use the DB object like this:
def getUsersDB(self):
cursor = db.get_cursor() # cursor from a live connection
cursor.execute("SELECT * FROM Users")
users = cursor.fetchall()
return users
import sshtunnel
import time
import logging
import mysql.connector
class SelectCommand():
def __init__(self,dbcmd,value = None, mul = False):
self.dbcmd = dbcmd
self.value = value
self.mul = mul
def execute(self):
try:
print("try1")
connection = mysql.connector.connect(
user='**myuser**', password='**pass**',
host='127.0.0.1', port=server.local_bind_port,
database='**myuser$test**', autocommit = True
)
print("try2")
connection.autocommit = True
mycursor = connection.cursor()
sql = self.dbcmd
val = self.value
mycursor.execute(sql, val)
myresult = mycursor.fetchone()
mycursor.close()
connection.close()
if myresult == None or self.mul == True:
return myresult
return myresult[0]
except Exception as e:
print(e)
return "server disconnect "
sshtunnel.SSH_TIMEOUT = 5.0
sshtunnel.TUNNEL_TIMEOUT = 5.0
def get_server():
#sshtunnel.DEFAULT_LOGLEVEL = logging.DEBUG
server = sshtunnel.SSHTunnelForwarder(
('ssh.pythonanywhere.com'),
ssh_username='**myuser**', ssh_password='**mypass**',
remote_bind_address=('**myuser.mysql.pythonanywhere-services.com**', 3306))
return server
server = get_server()
server.start()
while True :
if(server.is_active):
print("alive... " + (time.ctime()))
print(SelectCommand("SELECT * FROM A_table WHERE id = %s", (1,), mul = True).execute())
else:
print("reconnecting... " + time.ctime())
server.stop()
server = get_server()
server.start()
time.sleep(8)
Now i want use sshtunnel connect with database of pythonanywhere, and i want check connecting of sshtunnel if connect do select command else wait for new connecting. i try do Python: Automatically reconnect ssh tunnel after remote server gone down . but my problem is when i query database i try turn off my WIFI
my console show this message (Could not establish connection from ('127.0.0.1', 54466) to remote side of the tunnel) and Socket exception: An existing connection was forcibly closed by the remote host (10054) then it's result to stopping of my program. How can i fix.
Below code works perfectly in python2 with MySQLDB, how can I make it Python3 compatible?
I have debugged and searched for similar questions.
Error:
Exception ignored in: > Traceback (most recent call last): File > > > "/usr/local/lib/python3.4/dist-packages/pymysql/cursors.py", line 41, in del File > "/usr/local/lib/python3.4/dist-packages/pymysql/cursors.py", line 47, in close ReferenceError: weakly-referenced object no longer exists –
class Database():
def __init__(self):
self.host = 'localhost'
self.user = 'user'
self.password = 'pwd'
self.db = 'dbname'
self.connection = pymysql.connect(host=self.host, user=self.user, passwd=self.password, db=self.db,use_unicode=True, charset="utf8")
self.cursor = self.connection.cursor()
def storeToDB(self,ID,string,g,e):
import datetime
curr_time = datetime.datetime.now()
status = 0
try:
self.cursor.execute("""INSERT INTO master_data (`job_id`,`sstring`,`grl`,`erl`,`status`,`insert_timestamp`) \
VALUES (%s,%s,%s,%s,%s,%s)""",(jobID,search_string,g,e,status,curr_time))
self.connection.commit()
except:
self.connection.rollback()
if __name__ == "__main__":
db = Database()
db.storeToDB(20,"hello","something.com","e.com")
Try to do some opt before db operation finish :
cursor.close()
connection.close()
I'm trying use Python and pyodbc to access SQL server 2008. The first connection works. Then, after the program finishes its job, it closes the connection. When the program tries to access the database and connect to it again, it fails in the statement:
self.conn = pyodbc.connect(DRIVER=self.DRIVER, SERVER=self.SERVER, DATABASE=self.DATABASE, UID=self.UID, PWD=self.PWD, charset="UTF-8")
but the first time is OK. So does anyone know why? Below is the Python code:
class ODBC_MS:
def __init__(self, DRIVER,SERVER, DATABASE, UID, PWD):
''' initialization '''
self.DRIVER = DRIVER
self.SERVER = SERVER
self.DATABASE = DATABASE
self.UID = UID
self.PWD = PWD
def _GetConnect(self):
''' Connect to the DB '''
if not self.DATABASE:
raise(NameError,"no getting db name")
try:
self.conn = pyodbc.connect(DRIVER=self.DRIVER, SERVER=self.SERVER,
DATABASE=self.DATABASE, UID=self.UID,
PWD=self.PWD, charset="UTF-8")
except Exception,e:
print e.message
else:
self.cur = self.conn.cursor()
if not self.cur:
raise(NameError,"connected failed!")
else:
return self.cur, self.conn
def ExecNoQuery(self,conn, cursor, sql):
cursor.execute(sql)
ret = conn.commit()
return ret
def _UnConnect(self,conn, cursor):
conn.close()
if __name__ == '__main__':
ms = ODBC_MS('{SQL SERVER}', r'<server>', '<db>', '<user>', '<password>')
cursor, conn = ms._GetConnect() #connection
sql = "create table XX for example"
ret = ms.ExecNoQuery(conn, cursor,sql) #sql operation
ms._UnConnect(conn, cursor) #close db
#access the database the second time.
ms = ODBC_MS('{SQL SERVER}', r'<server>', '<db>', '<user>', '<password>')
cursor, conn = ms._GetConnect() # not success, I don't know why
sql = "create table XX for example"
ret = ms.ExecNoQuery(conn, cursor,sql) #sql operation
ms._UnConnect(conn, cursor) #close db
The second time when the program calls ms.GetConnect(), the statement self.conn = pyodbc.connect(DRIVER=self.DRIVER, SERVER=self.SERVER, DATABASE=self.DATABASE, UID=self.UID, PWD=self.PWD, charset="UTF-8") fails.