How to set connection with database in the api Flask? - python

I'm trying to create the api on my linux PC. At this moment I have support for some basic requests which were done just for testing. My api works in cooperation with uswgi+nginx+flask. And now I'm trying to add connection to the database. For this purpose I had installed MySQL and created database. But I don't understand how to connect from the api to database. For example here is code of the script which can connect to the DB but it works separately of the api:
try:
connection = mysql.connector.connect(host='localhost',
database='tired_db',
user='test',
password='pw')
if connection.is_connected():
mycursor = connection.cursor()
mycursor.execute("SHOW TABLES")
for x in mycursor:
print(x)
return connection
except Error as e:
print("Error while connecting to MySQL", e)
finally:
if connection.is_connected():
mycursor.close()
connection.close()
print("MySQL connection is closed")
and it works correctly. I thought that maybe I can call this connection like some metaclass:
import mysql.connector
from mysql.connector import Error
class DbProvider(type):
#property
def my_data(cls):
try:
connection = mysql.connector.connect(host='localhost',
database='tired_db',
user='test',
password='pw')
if connection.is_connected():
mycursor = connection.cursor()
mycursor.execute("SHOW TABLES")
for x in mycursor:
print(x)
return connection
except Error as e:
print("Error while connecting to MySQL", e)
finally:
if connection.is_connected():
mycursor.close()
connection.close()
print("MySQL connection is closed")
class MyClass(metaclass=DbProvider):
pass
if __name__ == "__main__":
MyClass.my_data
but I think that such stuff can be done with more efficient way. For example here is some request in the api:
#app.route("/api/login", methods = ['POST'])
def logIn():
return "all is ok"
and the idea is that for example I have to connect during this request to the DB and check whether a user exists or not and if all is ok generate+save some token to the database. I don't understand whether it is important to keep connection alive during all api uptime or only during requests. And also is it important to close connection after an every request or we have to keep alive it forever. And also how to call connection from separate class, or I have to have all stuff in one file together with api calls.


You can write a connection manager class that connects to the database, performs operations, and closes the connection.
import mysql.connector
from mysql.connector import Error
class ConnectionManager:
def __init__(self, host, database, user, password):
self.host = host
self.database = database
self.user = user
self.password = password
def __enter__(self):
try:
self.connection = mysql.connector.connect(
host=self.host,
database=self.database,
user=self.user,
password=self.password
)
if self.connection.is_connected():
self.cursor = self.connection.cursor()
return self.cursor
except Error as e:
print("Error while connecting to MySQL", e)
def __exit__(self, type, value, traceback):
if self.connection.is_connected():
self.cursor.close()
self.connection.close()
print("MySQL connection is closed")
#app.route("/api/login", methods = ['POST'])
def logIn():
with ConnectionManager('localhost', 'tired_db', 'test', 'pw') as cursor:
cursor.execute("SELECT * FROM users WHERE username = %s", ('test_user',))
result = cursor.fetchone()
if result:
# generate and save token
return "all is ok"
else:
return "user not found"

Related

What is the correct way to implement a context manager in psycopg2?

I've been refactoring my psycopg2 code using functions, previously I had it all on a try-except-finally block, however I'm not quite sure how to implement a context-manager to handle the connection and cursor. My SQL queries work and look like this:
def random_query(schema, table, username, number_of_files):
random_query = sql.SQL("SELECT * FROM {schema}.{table} WHERE username = {username} ORDER BY RANDOM() LIMIT {limit}").format(
schema=sql.Identifier(schema),
table=sql.Identifier(table),
username=sql.Literal(username),
limit=sql.Literal(number_of_files)
)
cursor.execute(random_query)
return cursor.fetchone()
def insert_query(schema, table, values):
insert_query = sql.SQL("INSERT INTO {schema}.{table}(shortcode, username, filename, extension) VALUES ({shortcode}, {username}, {filename}, {extension})").format(
schema=sql.Identifier(schema),
table=sql.Identifier(table),
shortcode=sql.Literal(values[0]),
username=sql.Literal(values[1]),
filename=sql.Literal(values[2]),
extension=sql.Literal(values[3])
)
cursor.execute(insert_query)
conn.commit()
First version:
#contextmanager
def get_connection():
connection = psycopg2.connect(**DB_CONNECTION)
try:
yield connection
except Exception as err:
connection.rollback()
print('Error: ', err)
raise
finally:
if (connection):
connection.close()
print("Connection is closed.")
#contextmanager
def get_cursor(connection):
cursor = connection.cursor()
try:
yield cursor
finally:
cursor.close()
with get_connection() as conn, get_cursor(conn) as cursor:
random_record = random_query('test_schema', 'test_table', 'username', 1)
insert_query('test_schema', 'test_table2', random_record)
Second version:
#contextmanager
def sql_connection():
connection = psycopg2.connect(**DB_CONNECTION)
cursor = connection.cursor()
try:
yield connection,cursor
except Exception as err:
connection.rollback()
print('Error : ', err)
raise
finally:
if (connection):
cursor.close()
connection.close()
print("Connection is closed")
with sql_connection() as (conn, cursor):
random_record = random_query('test_schema', 'test_table', 'username', 1)
insert_query('test_schema', 'test_table2', random_record)
My questions are:
Is there any difference between the first and the second version? Which one is preferable?
As you can see in insert_query, there is a line that calls conn.commit() From the documentation, I understand that this is not necessary if we are using a context manager. Can I remove them?
Changed in version 2.5: if the connection is used in a with statement,
the method is automatically called if no exception is raised in the
with block.

Neither version is preferable, you are still over complicating things by duplicating behavior. Per the example here Connection:
import psycopg2
connection = psycopg2.connect(**DB_CONNECTION)
with connection:
with connection.cursor() as cur:
cur.execute(<sql>)
with connection:
with connection.cursor() as cur:
cur.execute(<other_sql>)
Committing, rollback on the connection and closing of cursor is done for you. All you have to do is connection.close() when you no longer want to use the connection.

Python : threading.Thread block

import pymysql
from multiprocessing.pool import ThreadPool
def process(item):
host = str(item).strip()
db = None
try:
db = pymysql.connect(host=host, user='root', passwd='root', port=3306, connect_timeout=10,
write_timeout=10)
r = open("success.txt", 'a')
r.write(host)
r.write("\n")
pass
except Exception as msg:
print(msg)
pass
finally:
if db:
db.close()
pass
if __name__ == '__main__':
t = ThreadPool(100)
t.map_async(process,open('3306.txt').readlines())
t.close()
t.join()
I want to verify in batches whether the MySQL account passwords of some IPs are valid, but my code will have the problem of sub-thread blocking. In fact, the last IP has been verified, but the main thread has not ended. I don't know how to modify it. You can provide Some modifications and suggestions?

mysql-connector won't connect

import mysql.connector
from mysql.connector import Error
try:
connection = mysql.connector.connect(host='localhost',
database='Electronics',
user='pynative',
password='pynative##29')
if connection.is_connected():
db_Info = connection.get_server_info()
print("Connected to MySQL Server version ", db_Info)
cursor = connection.cursor()
cursor.execute("select database();")
record = cursor.fetchone()
print("You're connected to database: ", record)
except Error as e:
print("Error while connecting to MySQL", e)
finally:
if connection.is_connected():
cursor.close()
connection.close()
print("MySQL connection is closed")
I know this should work because I have the code copied from a website. But for some reason it show s the error
'if connection.is_connected():
NameError: name 'connection' is not defined'

If the mysql.connector.connect raisesan error, then connection variabe is never defined and you can't use it in the finally.
A solution is to initialize it as None and check that it isn't none before using it in the finally
connection = None
try:
connection = mysql.connector.connect(host='localhost',
database='Electronics',
user='pynative',
password='pynative##29')
if connection.is_connected():
db_Info = connection.get_server_info()
print("Connected to MySQL Server version ", db_Info)
cursor = connection.cursor()
cursor.execute("select database();")
record = cursor.fetchone()
print("You're connected to database: ", record)
except Error as e:
print("Error while connecting to MySQL", e)
finally:
if connection and connection.is_connected():
cursor.close()
connection.close()
print("MySQL connection is closed")

Python mysql (using pymysql) auto reconnect

I'm not sure if this is possible, but I'm looking for a way to reconnect to mysql database when the connection is lost. All the connections are held in a gevent queue but that shouldn't matter I think. I'm sure if I put some time in, I can come up with a way to reconnect to the database. However I was glancing pymysql code and I saw that there is a 'ping' method in Connection class, which I'm not sure exactly how to use.
The method looks like it will reconnect first time but after that it switched the reconnect flag to False again? Can I use this method, or is there a different way to establish connection if it is lost? Even if it is not pymysql how do people tackle, database servers going down and having to re-establish connection to mysql server?
def ping(self, reconnect=True):
''' Check if the server is alive '''
if self.socket is None:
if reconnect:
self._connect()
reconnect = False
else:
raise Error("Already closed")
try:
self._execute_command(COM_PING, "")
return self._read_ok_packet()
except Exception:
if reconnect:
self._connect()
return self.ping(False)
else:
raise

Well, I've got the same problem in my application and I found a method on the PyMySQL documentation that pings to the server and check if the connection was closed or not, if it was closed, then it reconnects again.
from pymysql import connect
from pymysql.cursors import DictCursor
# create the connection
connection = connect(host='host', port='port', user='user',
password='password', db='db',
cursorclass=DictCursor)
# get the cursor
cursor = connection.cursor()
# if the connection was lost, then it reconnects
connection.ping(reconnect=True)
# execute the query
cursor.execute(query)
I hope it helps.

Finally got a working solution, might help someone.
from gevent import monkey
monkey.patch_socket()
import logging
import gevent
from gevent.queue import Queue
import pymysql as db
logging.basicConfig(level=logging.DEBUG)
LOGGER = logging.getLogger("connection_pool")
class ConnectionPool:
def __init__(self, db_config, time_to_sleep=30, test_run=False):
self.username = db_config.get('user')
self.password = db_config.get('password')
self.host = db_config.get('host')
self.port = int(db_config.get('port'))
self.max_pool_size = 20
self.test_run = test_run
self.pool = None
self.time_to_sleep = time_to_sleep
self._initialize_pool()
def get_initialized_connection_pool(self):
return self.pool
def _initialize_pool(self):
self.pool = Queue(maxsize=self.max_pool_size)
current_pool_size = self.pool.qsize()
if current_pool_size < self.max_pool_size: # this is a redundant check, can be removed
for _ in xrange(0, self.max_pool_size - current_pool_size):
try:
conn = db.connect(host=self.host,
user=self.username,
passwd=self.password,
port=self.port)
self.pool.put_nowait(conn)
except db.OperationalError, e:
LOGGER.error("Cannot initialize connection pool - retrying in {} seconds".format(self.time_to_sleep))
LOGGER.exception(e)
break
self._check_for_connection_loss()
def _re_initialize_pool(self):
gevent.sleep(self.time_to_sleep)
self._initialize_pool()
def _check_for_connection_loss(self):
while True:
conn = None
if self.pool.qsize() > 0:
conn = self.pool.get()
if not self._ping(conn):
if self.test_run:
self.port = 3306
self._re_initialize_pool()
else:
self.pool.put_nowait(conn)
if self.test_run:
break
gevent.sleep(self.time_to_sleep)
def _ping(self, conn):
try:
if conn is None:
conn = db.connect(host=self.host,
user=self.username,
passwd=self.password,
port=self.port)
cursor = conn.cursor()
cursor.execute('select 1;')
LOGGER.debug(cursor.fetchall())
return True
except db.OperationalError, e:
LOGGER.warn('Cannot connect to mysql - retrying in {} seconds'.format(self.time_to_sleep))
LOGGER.exception(e)
return False
# test (pytest compatible) -------------------------------------------------------------------------------------------
import logging
from src.py.ConnectionPool import ConnectionPool
logging.basicConfig(level=logging.DEBUG)
LOGGER = logging.getLogger("test_connection_pool")
def test_get_initialized_connection_pool():
config = {
'user': 'root',
'password': '',
'host': '127.0.0.1',
'port': 3305
}
conn_pool = ConnectionPool(config, time_to_sleep=5, test_run=True)
pool = conn_pool.get_initialized_connection_pool()
# when in test run the port will be switched back to 3306
# so the queue size should be 20 - will be nice to work
# around this rather than test_run hack
assert pool.qsize() == 20

The easiest way is to check the connection right before sending a query.
You can do this by creating a small class that contains two methods: connect and query:
import pymysql
import pymysql.cursors
class DB:
def connect(self):
self.conn = pymysql.connect(
host=hostname,
user=username,
password=password,
db=dbname,
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor,
port=3306)
def query(self, sql):
try:
cursor = self.conn.cursor()
cursor.execute(sql)
except pymysql.OperationalError:
self.connect()
cursor = self.conn.cursor()
cursor.execute(sql)
return cursor
db = DB()
Now, whenever you send a query using db.query("example SQL") the request is automatically prepared to encounter a connection error and reconnects using self.connect() if it needs to.
Remember: This is a simplified example. Normally, you would want to let PyMySQL help you escape special characters in your queries. To do that, you would have to add a 2nd parameter in the query method and go from there.

the logic is quite simple, if connection close then try to reconnect for several times in this case I use max tries for 15 times to reconnect or ping.
import pymysql, pymysql.cursors
conn = pymysql.connect(
host=hostname,
user=username,
password=password,
db=dbname,
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor,
)
cursor = conn.cursor()
# you can do transactions to database and when you need conn later, just make sure the server is still connected
if conn.open is False:
max_try = 15
try = 0
while conn.open is False:
if try < max_try:
conn.ping() # autoreconnect is true by default
try +=1
# check the conn again to make sure it connected
if conn.open:
# statements when conn is successfully reconnect to the server
else:
# it must be something wrong : server, network etc

Old but I encountered a similar problem for accessing hosted db within programs. The solution I ended up using was to create a decorator to automatically reconnect when making a query.
given a connection function:
def connect(self):
self.conn = mysql.connector.connect(host=self.host, user=self.user,
database=self.database, password=self.password)
self.cursor = self.conn.cursor()
print("Established connectionn...")
I created
def _reconnect(func):
#wraps(func)
def rec(self,*args,**kwargs):
try:
result = func(self,*args,**kwargs)
return result
except (mysql.connector.Error, mysql.connector.Warning) as e:
self.connect()
result = func(self,*args,**kwargs)
return result
return rec
Such that any function using the connection can now be decorated as so
#_reconnect
def check_user_exists(self,user_id):
self.cursor.execute("SELECT COUNT(*) FROM _ where user_id={};".format(user_id))
if self.cursor.fetchall()[0][0]==0:
return False
else:
return True
This decorator will re-establish a connection and rerun any function involving a query to the db.

You can use a property to keep the connection alive every time you do querying:
import pymysql
import pymysql.cursors
import pandas as pd
class DB:
def __init__(self, hostname='1.1.1.1', username='root', password='password',
database=None, port=3306, charset="utf8mb4"):
self.hostname = hostname
self.database = database
self.username = username
self.password = password
self.port = port
self.charset = charset
self.connect()
#property
def conn(self):
if not self.connection.open:
print('Going to reconnect')
self.connection.ping(reconnect=True)
return self.connection
def connect(self):
self.connection = pymysql.connect(
host=self.hostname,
user=self.username,
password=self.password,
db=self.database,
charset=self.charset,
cursorclass=pymysql.cursors.DictCursor,
port=self.port)
def query(self, sql):
return pd.read_sql_query(sql, con=self.conn)
db = DB(hostname='1.1.1.1', username='root', password='password', database=None, port=3306, charset="utf8mb4")

Making sure a query gets executed

I have a server which executes properly 9/10 times, but sometimes I get the error “Lost connection to MySQL server during query” and then the whole process stops/freezes.
I'm trying to let a MySQL query execute again if it fails using a function for this, unfortunately it seems that the code 'stops' or get stuck whenever I get an error. Am I doing something wrong?
Here string is something like: SELECT xx FROM xxx WHERE xxx
try:
db = MySQLdb.connect (host = "",
user = "",
passwd = "",
db = "" )
except MySQLdb.Error, e:
print("Error %d: %s" %(e.args[0], e.args[1]))
sys.exit(1);
cursor = db.cursor()
def mysql_handling(string):
while 1:
try:
cursor.execute(string)
if 'SELECT' not in string:
db.commit()
break
except:
mysql_error_tracking(string)
mysql_error_tracking is a function to monitor what queries fail most of the time (not relevant).

Add a timeout to your connection method:
db = MySQLdb.connect (host = HOST, user = USER, passwd = PASS, db = DB,
connect_timeout = TIMEOUT)
Do this in your except block so that you do a re-connect.
To make your life easier, place your connection code in a separate function so you can re-use it. Something like this (untested):
def get_cursor()
try:
db = MySQLdb.connect (host = HOST, user = USER, passwd = PASS, db = DB,
connect_timeout = TIMEOUT)
except MySQLdb.Error, e:
print("Error %d: %s" %(e.args[0], e.args[1]))
sys.exit(1);
return db.cursor()
def mysql_handling(cursor, string):
while True:
try:
cursor.execute(string)
if 'SELECT' not in string:
db.commit()
break
except MySQLdb.MySQLError:
cursor.close()
mysql_error_tracking(string)
cursor = get_cursor()
def main():
mysql_handling(get_cursor(), string)

Categories

Resources