Develop Reference

Psycopg3 returns None randomly

So i have a flask app, that connects to a Postgres12 DB using a user that just has select privileges. Often times i see my apis return 400 error, and api does this when the sql query executed returns None.
I have built a small wrapper class over execute and executemany function for some error handling.
import time
from functools import wraps
import psycopg
from psycopg import InterfaceError, OperationalError
logger = logging.getLogger(__name__)
def retry(fn):
#wraps(fn)
def wrapper(*args, **kw):
cls = args[0]
exec = None
for x in range(cls._reconnectTries):
try:
return fn(*args, **kw)
except (InterfaceError, OperationalError) as e:
logger.warning(f"Database Connection {e} exception type: {type(e)}")
logger.info(f"Idle for {cls._reconnectIdle} seconds")
time.sleep(cls._reconnectIdle)
cls._connect()
exec = e
import sys
logger.exception(f"Exiting the system, {exec} ")
sys.exit(exec)
return wrapper
class Connection:
_reconnectTries = 5
_reconnectIdle = 2
def __init__(self, conn_string):
self._conn_string = conn_string
self.conn = None
self.cursor = None
self._connect()
def _connect(self):
self.conn = psycopg.connect(self._conn_string)
self.conn.autocommit = True
self.cursor = self.conn.cursor()
#retry
def execute(self, **kwargs):
# self.conn.commit()
if "query" in kwargs:
"""
this is done to ensure postgres logs multi line queries sent by client in single line for easier
log collection and debugging.
"""
kwargs["query"] = kwargs["query"].replace("\n", " ")
kwargs["query"] = " ".join(kwargs["query"].split())
return self.cursor.execute(**kwargs)
#retry
def executemany(self, **kwargs):
# self.conn.commit()
return self.cursor.executemany(**kwargs)
For the sake of simplicity, the query code looks somewhat like
store_detail_by_link_query = """
SELECT
json_build_object('id', store.id, 'uuid', store.uuid)
FROM
optimus_store store
WHERE
store.link= %(store_link_or_domain)s and store.is_active = TRUE and store.is_deleted = FALSE;
"""
optimus_connection = Connection(conn_string=CONN_STRING)
params = {
"store_link_or_domain": "dipen28",
}
row = optimus_connection.execute(
query=store_detail_by_link_query,
params=params,
).fetchone()
The problem is, in the same api call if i just do a check that result is None, then rerun the query, at that time the result comes.
I know the data is there in database, this is coming to us in our production system and i am unable to reproduce it at local.
Any help is much appreciated.

Library program in Python 3 (beginner) with psycopg2

I need help with this program. I am making a program for a library in Python 3 (psycopg2) and I am making my modules to handle my tables, I have already done the "create" module and I am doing the "delete" module, I need help to do it, the code used is the following:
This is my class ConnectioDB: (it works to connect to my database in postgresql)
class ConnectionDB:
"""Connection class."""
bd = None
cursor = None
def __init__(self, **param):
"""Connection constructor."""
try:
self.db = connect(
host = '127.0.0.1', # localhost
user = 'postgres'
password = #$#!#*
database = 'national-library'
)
self.cursor = self.db.cursor()
except Error as e:
write_errors(e, 'Failed to connect to database')
with the following lines are used to execute sql code:
def execute_sql(
self,
sentence_sql,
param=None,
write_in_db=True
):
"""Execute SQL code."""
try:
execute = self.cursor.execute(sentence_sql, param)
if write_in_db:
result = self.db.commit()
except Exception as e:
write_errors(e, f"An error occurred while executing the SQL statement:\n\n{sentence_sql}\n")
if write_in_db:
self.db.rollback()
now this is my Model class that has the "create" module
class Model():
"""Generic model class."""
table_name = None
connection = ConnectionDB()
def create(self):
"""Save in database."""
table_name = self.table_name
keys = ", ".join(self.__dict__.keys())
values_placeholders = ", ".join(["%s" for i in range(len(self.__dict__.keys()))])
values = self.__dict__.values()
sql = f"INSERT INTO {table_name} ({keys}) VALUES ({values_placeholders})"
self.connection.execute_sql(sql, tuple(values))
I was trying to do the following code for my "delete" module but I'm not sure if it's ok:
def delete(self, column_id):
"""Delete an item in the database."""
table_name = self.table_name
sql = f"DELETE FROM {table_name} WHERE id = %s"
self.connection.execute_sql(sql, (column_id, ))
I hope you can help me. Thank you!

Use decorator to connect to a postgres database

I am working on a program to store my picture meta data and thumbnails into a postgres database using python and psycopg2. In the example I have defined a class MyDbase with methods to create a table, store a value and load a value. Each of these methods needs to connect to the database and a cursor object to execute sql commands. To avoid repetition of code to make the connection and get the cursor I have made a sub class DbDecorators with a decorator connect.
My question: is this a proper way to handle this and specifically using the with statement and passing the cursor to the Dbase method (func) inside the wrapper?
from functools import wraps
import psycopg2
class MyDbase:
''' example using a decorator to connect to a dbase
'''
table_name = 'my_table'
class DbDecorators:
host = 'localhost'
db_user = 'db_tester'
db_user_pw = 'db_tester_pw'
database = 'my_database'
#classmethod
def connect(cls, func):
#wraps(func)
def wrapper(*args, **kwargs):
connect_string = f'host=\'{cls.host}\' dbname=\'{cls.database}\''\
f'user=\'{cls.db_user}\' password=\'{cls.db_user_pw}\''
result = None
try:
with psycopg2.connect(connect_string) as connection:
cursor = connection.cursor()
result = func(*args, cursor, **kwargs)
except psycopg2.Error as error:
print(f'error while connect to PostgreSQL {cls.database}: '
f'{error}')
finally:
if connection:
cursor.close()
connection.close()
print(f'PostgreSQL connection to {cls.database} is closed')
return result
return wrapper
#staticmethod
def get_cursor(cursor):
if cursor:
return cursor
else:
print(f'no connection to database')
raise()
#classmethod
#DbDecorators.connect
def create_table(cls, *args):
cursor = cls.DbDecorators().get_cursor(*args)
sql_string = f'CREATE TABLE {cls.table_name} '\
f'(id SERIAL PRIMARY KEY, name VARCHAR(30));'
print(sql_string)
cursor.execute(sql_string)
#classmethod
#DbDecorators.connect
def store_value(cls, name, *args):
cursor = cls.DbDecorators().get_cursor(*args)
sql_string = f'INSERT INTO {cls.table_name} (name) VALUES (%s);'
print(sql_string)
cursor.execute(sql_string, (name,))
#classmethod
#DbDecorators.connect
def load_value(cls, _id, *args):
cursor = cls.DbDecorators().get_cursor(*args)
sql_string = f'SELECT * FROM {cls.table_name} where id = \'{_id}\';'
print(sql_string)
cursor.execute(sql_string)
db_row = cursor.fetchone()
return db_row
def test():
my_db = MyDbase()
my_db.create_table()
my_db.store_value('John Dean')
db_row = my_db.load_value(1)
print(f'id: {db_row[0]}, name: {db_row[1]}')
if __name__ == '__main__':
test()

probably I did not get your request correctly. Why you need decorator but don't use context manager? Like define db client in any file where from you can import it later and then use it in context manager –
from psycopg2 import SomeDataBase
db = SomeDataBase(credentials)
def create_table(table_name):
with db:
sql_string = f'CREATE TABLE {table_name} '\
f'(id SERIAL PRIMARY KEY, name VARCHAR(30));'
db.cursor.execute(sql_string)

Using a context manager will not close the connection, only the cursor. So using the decorator pattern actually makes more sense here. More info on the context manager: https://www.psycopg.org/docs/usage.html (scroll down to the "with statement" section.)

Python: Create a class for SQL select and update statement

Is there any other way to improve this code? I'm trying to create a class with select and update method then I call it for future use.
connect.py
import pymysql
class SQL:
def __init__(self):
self.cnx = pymysql.connect(host...)
self.c = self.cnx.cursor()
def select(self, sql):
self.c.execute(sql)
self.sel = self.c.fetchone()
self.c.close()
self.cnx.close()
return self.sel
def update(self, sql):
self.c.execute(sql)
self.upd = self.cnx.commit()
self.c.close()
self.cnx.close()
return self.upd
from connect import SQL
stmt = "SELECT * FROM name;"
result = SQL().select(stmt)

import pymysql
class Connection:
def __init__(self):
self.con = pymysql.connect(
host = 'localhost',
port = 3306,
user = 'user',
password = 'password',
db = 'dbname',
autocommit=True,
charset = 'utf8mb4',
cursorclass = pymysql.cursors.DictCursor
)
self.cur = self.con.cursor()
def _select(self,sql,args=None):
self.cur.execute(sql,args)
self.sel = self.cur.fetchone()
self.cur.close()
self.con.close()
return self.sel
def _selectAll(self,sql,args=None):
self.cur.execute(sql,args)
self.sel = self.cur.fetchall()
self.cur.close()
self.con.close()
return self.sel
def _insert(self,sql,args=None):
self.ins = self.cur.executemany(sql,args)
return self.ins
def _update(self,sql, args=None):
self.upd = self.cur.executemany(sql,args)
return self.upd
def _delete(self, sql, args=None):
self.delete = self.cur.executemany(sql,args)
return self.delete
I used a logic to repository creating this class, an example(with PyPika) of use is this:
from foo.bar import Connection
from pypika import Query, Table
users = Table("users")
class UserRepository(Connection):
def getById(self, args=None):
query = Query.from_(users).select("*").where(users.tg_id == "%s")
q = query.get_sql(quote_char=None)
return self._select(q, args)
And now an example of how to invoke the repository:
from foo.bar import UserRepository
def init(user):
row = UserRepository().getById(user)
for i in row:
print(i)

Add a function in this class ,function close().use close() in the end of the main().Besides ,use try and except in your every function. Also you can add a reinit function to be called in every function.

Write a database class in python

For a hobby project I'm doing at the moment I want to write an object oriented python program.But one of the first issue I'm running into is creating a class for my (MySQL) database.
I am trying to use as few packages as possible and I want to try to write the database class by using only pymysql.The problem is that there is an overwhelming amount of libraries and a lack of explanation on writing a good database class in python.
Any suggestions and especially examples would be much appreciated

When having the exact same situation, I found the mysql-connector-python class with which I created a small "model" to call from other classes. This is a cut-down version which shows various db calls. As you can see I have a config class which holds all db authentication info and more).
# dependancy: mysql-connector-python (https://dev.mysql.com/downloads/connector/python/2.1.html)
import mysql.connector
import time
import config
import HTMLParser
import StringIO
html_parser = HTMLParser.HTMLParser()
try:
connection = mysql.connector.connect( user=config.DB_USER, password=config.DB_PASSWORD,
host = config.DB_HOST, database=config.DB_DATABASE, unix_socket=config.UNIX_SOCKET)
cursor = connection.cursor()
except mysql.connector.Error as err:
logger.log('Database connection failed for '+config.DB_USER+'#'+config.DB_HOST+'/'+config.DB_DATABASE)
exit()
def get_bad_words():
sql = ("SELECT word FROM word_blacklist")
results = execute(sql)
return results
def get_moderation_method():
sql = ("SELECT var_value FROM settings "
"WHERE var_key = %(key)s")
results = execute(sql, True, {'key':'moderation_method'})
return results[0]
def current_events():
sql = ("SELECT count(id) FROM events WHERE event_date >= DATE_SUB(NOW(), INTERVAL 2 hour) AND event_date <= DATE_ADD(NOW(), INTERVAL 5 hour)")
results = execute(sql, True)
return results[0]
def insert_social_post(channel, filter_type, post_id, validate, user_name, user_id, user_profile_picture, text, post_date, image_url, state):
try:
san_user_name = html_parser.unescape(user_name.encode('utf-8').strip()).decode("utf8").encode('ascii','ignore')
except:
san_user_name = html_parser.unescape(user_name.strip())
try:
san_text = html_parser.unescape(text.encode('utf-8').strip()).decode("utf8").encode('ascii','ignore')
except:
san_text = html_parser.unescape(text.strip())
insert_post = ("INSERT IGNORE INTO social_posts "
"(channel, filter_type, post_id, validate, user_name, user_id, user_profile_picture, text, post_date, image_url, state)"
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)")
execute(insert_post, False, [channel, filter_type, str(post_id), validate,
san_user_name.strip(), user_id, user_profile_picture, san_text.strip(), post_date, image_url, state], True)
def delete_posts(ids):
fmt = ','.join(['%s'] * len(ids))
cursor.execute("DELETE FROM `social_posts` WHERE id IN (%s)" % fmt,
tuple(ids))
connection.commit()
def update_campaigns(campaigns):
sql = ("UPDATE social_campaigns "
"SET last_updated = NOW()"
"WHERE id IN ("+(','.join(str(c) for c in campaigns))+")")
execute(sql, False, None, True)
def execute(tuple, single = False, args = {}, commit = False):
cursor.execute(tuple, args)
if commit == True:
connection.commit()
else:
if single == True:
return cursor.fetchone()
else:
return cursor.fetchall()
def lastrowid():
return cursor.lastrowid
def close():
connection.close()
Call upon the class like this:
import db
bad_words = db.get_bad_words()

import sqlite3
"""singleton class to deal with db"""
'''same can be use for pymysql just replace the sqlite3 with pymysql'''
class DBConnection:
instance = None
def __new__(cls, *args, **kwargs):
if cls.instance is None:
cls.instance = super().__new__(DBConnection)
return cls.instance
return cls.instance
def __init__(self, db_name='you-db-name'):
self.name = db_name
# connect takes url, dbname, user-id, password
self.conn = self.connect(db_name)
self.cursor = self.conn.cursor()
def connect(self):
try:
return sqlite3.connect(self.name)
except sqlite3.Error as e:
pass
def __del__(self):
self.cursor.close()
self.conn.close()
# write your function here for CRUD operations