I am trying to connect to ClearDb with Heroku from python app I use mysql.connector
My code look like this:
conn = mysql.connector.connect(
host="clearDbHost",
user="123qwe",
password="123qwe",
database="heroku_4fsdfsdf30daf8",
port=3306,
autocommit = True
)
curs = conn.cursor()
Is connection crating the tables, but after this is disconnect and can't execute curs.execute() command later in the code I got this error:
Traceback (most recent call last):
File "/app/.heroku/python/lib/python3.10/site-packages/mysql/connector/connection_cext.py", line 535, in cmd_query
self._cmysql.query(query,
_mysql_connector.MySQLInterfaceError: Lost connection to MySQL server during query
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File "/app/twitter.py", line 855,
in
getMyFollowersToDatabse() File "/app/twitter.py", line 186, in getMyFollowersToDatabse
curs.execute("INSERT INTO users (username, follow_id) VALUES ('" + user.username + "', '" + user.id + "')") File
"/app/.heroku/python/lib/python3.10/site-packages/mysql/connector/cursor_cext.py",
line 269, in execute
result = self._cnx.cmd_query(stmt, raw=self._raw, File "/app/.heroku/python/lib/python3.10/site-packages/mysql/connector/connection_cext.py",
line 540, in cmd_query
raise errors.get_mysql_exception(exc.errno, msg=exc.msg, mysql.connector.errors.OperationalError: 2013 (HY000): Lost connection
to MySQL server during query
I see in the Clear Db like is havin ?reconnect=true in the end of the database name.I am not using curs.close()
Related
I have a Python script which connects to my database, gets all the users' hashes and their emails, then parses through those hashes and gets some other data from the DB based on the user's hash value.
The problem is, that my MySQL Python connector breaks at various points and gives me this exception:
Traceback (most recent call last):
File "/home/antonio/.local/lib/python3.9/site-packages/mysql/connector/network.py", line 509, in open_connection
self.sock.connect(sockaddr)
socket.timeout: timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/antonio/.local/lib/python3.9/site-packages/flask/app.py", line 2447, in wsgi_app
response = self.full_dispatch_request()
File "/home/antonio/.local/lib/python3.9/site-packages/flask/app.py", line 1952, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/home/antonio/.local/lib/python3.9/site-packages/flask/app.py", line 1821, in handle_user_exception
reraise(exc_type, exc_value, tb)
File "/home/antonio/.local/lib/python3.9/site-packages/flask/_compat.py", line 39, in reraise
raise value
File "/home/antonio/.local/lib/python3.9/site-packages/flask/app.py", line 1950, in full_dispatch_request
rv = self.dispatch_request()
File "/home/antonio/.local/lib/python3.9/site-packages/flask/app.py", line 1936, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/home/antonio/Desktop/PythonProjects/crypton-portfolio-api/crypto_tracking_coins/main_with_flask.py", line 701, in get_all_users_with_portfolio_and_accounting_data
user_coins_from_db = perform_db_query_fetchall('SELECT `portfolio`, `date`, `type`, `invested` FROM `users` WHERE `hash` = %s AND `fake` = "no"', (user_hash, ))
File "/home/antonio/Desktop/PythonProjects/crypton-portfolio-api/crypto_tracking_coins/helpers.py", line 6, in perform_db_query_fetchall
mydb = mysql.connector.connect(
File "/home/antonio/.local/lib/python3.9/site-packages/mysql/connector/__init__.py", line 179, in connect
return MySQLConnection(*args, **kwargs)
File "/home/antonio/.local/lib/python3.9/site-packages/mysql/connector/connection.py", line 95, in __init__
self.connect(**kwargs)
File "/home/antonio/.local/lib/python3.9/site-packages/mysql/connector/abstracts.py", line 716, in connect
self._open_connection()
File "/home/antonio/.local/lib/python3.9/site-packages/mysql/connector/connection.py", line 206, in _open_connection
self._socket.open_connection()
File "/home/antonio/.local/lib/python3.9/site-packages/mysql/connector/network.py", line 511, in open_connection
raise errors.InterfaceError(
mysql.connector.errors.InterfaceError: 2003: Can't connect to MySQL server on 'localhost:3306' (timed out)
My helpers.py file, which includes all the logic for performing SQL queries, looks like this:
import mysql.connector
from config import *
def perform_db_query_fetchall(query: str, params: tuple):
# Initiate DB connection
mydb = mysql.connector.connect(
host=DatabaseConfig.host_db,
user=DatabaseConfig.user_db,
password=DatabaseConfig.password_db,
database=DatabaseConfig.database_db
)
# Initiate DB cursor
c = mydb.cursor()
# Execute SQL query and get the results
c.execute(query, params)
results = c.fetchall()
# Close the DB connection
c.close()
mydb.close()
# Return the results
return results
def perform_db_query_fetchone(query: str, params: tuple):
# Initiate DB connection
mydb = mysql.connector.connect(
host=DatabaseConfig.host_db,
user=DatabaseConfig.user_db,
password=DatabaseConfig.password_db,
database=DatabaseConfig.database_db
)
# Initiate DB cursor
c = mydb.cursor()
# Execute SQL query and get the results
c.execute(query, params)
results = c.fetchone()
# Close the DB connection
c.close()
mydb.close()
# Return the results
return results
def perform_db_query_with_commit(query: str, params: tuple):
# Initiate DB connection
mydb = mysql.connector.connect(
host=DatabaseConfig.host_db,
user=DatabaseConfig.user_db,
password=DatabaseConfig.password_db,
database=DatabaseConfig.database_db,
autocommit=True
)
# Initiate DB cursor
c = mydb.cursor()
# Execute SQL query and get the results
c.execute(query, params)
# Close the DB connection
c.close()
mydb.close()
return
I'm looping with a for loop (about 1900 iterations of the loop) and doing approximately as twice as many SQL queries in that for loop.
It always ends not finishing the job that I need the script to do with just printing out the above mentioned Connection Timed Out exception.
Interesting fact is, that it always crashes on different stage. The last I tried, it failed on the 1600th~ iteration. Sometimes it fails on the 30th~ iteration...
Any ideas what can I do to fix it?
Thank you.
This can happen if the database server is running out of free space. In my case the database server was full.
A MySQL server is initialized in Flask (with connexion) on service startup.
service.app.datastore = DatastoreMySQL(service.config)
class DatastoreMySQL(Datastore):
def __init__(self, config):
...
self.connection_pool = pooling.MySQLConnectionPool(
database=self.database,
host=self.hostname,
username=self.username,
password=self.password,
pool_name="pool_name",
pool_size=self.pool_size,
autocommit=True
)
def exec_query(self, query, params=None):
try:
connection = self.connection_pool.get_connection()
connection.ping(reconnect=True)
with closing(connection.cursor(dictionary=True, buffered=True)) as cursor:
if params:
cursor.execute(query, params)
else:
cursor.execute(query)
finally:
connection.close()
The view functions use the database by passing the DB reference from current_app.
def new():
do_something_in_db(current_app.datastore, request.get_json())
def do_something_in_db(db, data):
db.create_new_item(data)
...
However, a background process (run with APScheduler) must also run do_something_in_db(), but when passed a datastore reference an mysql.connector.errors.OperationalError error is thrown.
My understanding is that this error comes from two sources:
The server timed out and closed the connection. However, in this service the exec_query() function obtains a connection and executes right away, so there should be no reason that it times out. The monitor is also initialized at service startup with a datastore reference, but I am not sure how that can time out given that a new connection is created each time exec_query() is called.
The server dropped an incorrect or too large packet. However, there are no packets here - the process is run by a local background scheduler.
The error in full:
Job "Monitor.monitor_running_queries" raised an exception
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/mysql/connector/connection_cext.py", line 509, in cmd_query
raw_as_string=raw_as_string)
_mysql_connector.MySQLInterfaceError: MySQL server has gone away
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/k8s-service/lib/datastore/datastore_mysql.py", line 88, in exec_query
cursor.execute(query, params)
File "/usr/local/lib/python3.6/site-packages/mysql/connector/cursor_cext.py", line 276, in execute
raw_as_string=self._raw_as_string)
File "/usr/local/lib/python3.6/site-packages/mysql/connector/connection_cext.py", line 512, in cmd_query
sqlstate=exc.sqlstate)
mysql.connector.errors.DatabaseError: 2006 (HY000): MySQL server has gone away
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/apscheduler/executors/base.py", line 125, in run_job
retval = job.func(*job.args, **job.kwargs)
File "/opt/k8s-service/lib/background.py", line 60, in monitor_running_queries
self.handle_process_state(query.id, datastore, hive)
File "/opt/k8s-service/lib/background.py", line 66, in handle_process_state
query = datastore.get_item(query_id)
File "/opt/k8s-service/lib/datastore/datastore.py", line 48, in get_item
return_results=True)
File "/opt/k8s-service/lib/datastore/datastore.py", line 97, in exec_query
connection.close()
File "/usr/local/lib/python3.6/site-packages/mysql/connector/pooling.py", line 131, in close
cnx.reset_session()
File "/usr/local/lib/python3.6/site-packages/mysql/connector/connection_cext.py", line 768, in reset_session
raise errors.OperationalError("MySQL Connection not available.")
mysql.connector.errors.OperationalError: MySQL Connection not available.
When I try to run my code i get this error:
File "script.py", line 36, in <module>
result = cursor.execute(sql_insert_query)
File "/home/rexio18/.local/lib/python2.7/site-packages/mysql/connector/cursor_cext.py", line 266, in execute
raw_as_string=self._raw_as_string)
File "/home/rexio18/.local/lib/python2.7/site-packages/mysql/connector/connection_cext.py", line 405, in cmd_query
errno=2055, values=(addr, 'Connection not available.'))
mysql.connector.errors.OperationalError: 2055: Lost connection to MySQL server at 'localhost:3306', system error: Connection not available.
Code:
for child in root:
for element in child:
for subelement in element:
a = subelement.attrib['currency']
b = subelement.text
connection = mysql.connector.connect(host ='localhost', user ='root', passwd ='admin' , database ='python')
cursor = connection.cursor()
sql_insert_query = "INSERT INTO valoare(moneda, flux) VALUES (%s, %s)",(a, b)
result = cursor.execute(sql_insert_query)
connection.commit()
Here is code :
#!/usr/bin/env python
import pyhs2
try:
with pyhs2.connect(host='localhost',
port=10001,
authMechanism="PLAIN",
user='root',
password='test',
database='test') as conn:
with conn.cursor() as cur:
#Show databases
print cur.getDatabases()
#Execute query
cur.execute("select * from raw_stats")
#Return column info from query
print cur.getSchema()
#Fetch table results
for i in cur.fetch():
print i
except Thrift.TException, tx:
print '%s' % (tx.message)
Error!
Traceback (most recent call last): File "/usr/local/py/test.py", line
8, in database='default') as conn: File
"/usr/lib/python2.6/site-packages/pyhs2/init.py", line 7, in
connect
return Connection(*args, **kwargs) File "/usr/lib/python2.6/site-packages/pyhs2/connections.py", line 46, in
init
transport.open() File "/usr/lib/python2.6/site-packages/pyhs2/cloudera/thrift_sasl.py", line
55, in open
self._trans.open() File "/usr/lib64/python2.6/site-packages/thrift/transport/TSocket.py", line
101, in open
message=message) thrift.transport.TTransport.TTransportException: Could not connect to localhost:10001
It resolved by starting hiveServer2 service and changing the port 10000.
I have a python script I am running to add a document to a collection for a remote mongodb database. The file is as follows:
from pymongo import Connection
ip = 'x.x.x.x' #edited out
conn = Connection()
db = conn['netmon']
users = db.users
print 'number of users: ' + str(users.count())
if users.count() == 0:
print 'Please create a new account.'
t_user = raw_input('Username:')
users.insert({'username':unicode(t_user)})
conn.close()
When I run this script, I have some interesting behavior. In the server logs, it appears that the connection is accepted, the query is run, and then the connection is closed.
# Mongodb logs
14:27:18 [initandlisten] connection accepted from 108.93.46.75:39558 #1
14:27:18 [conn1] run command admin.$cmd { ismaster: 1 }
14:27:18 [conn1] command admin.$cmd command: { ismaster: 1 } ntoreturn:1 reslen:71 0ms
14:27:18 [conn1] run command netmon.$cmd { count: "users", fields: null, query: {} }
14:27:18 [conn1] Accessing: netmon for the first time
14:27:18 [conn1] command netmon.$cmd command: { count: "users", fields: null, query: {} } ntoreturn:1 reslen:58 10ms
14:27:18 [conn1] end connection 108.93.46.75:39558
However, this is what I get from the python script:
# Python error
> python testmongo.py
Traceback (most recent call last):
File "testmongo.py", line 7, in <module>
conn = Connection()
File "/usr/local/lib/python2.7/dist-packages/pymongo-2.2.1-py2.7-linux-x86_64.egg/pymongo/connection.py", line 290, in __init__
self.__find_node()
File "/usr/local/lib/python2.7/dist-packages/pymongo-2.2.1-py2.7-linux-x86_64.egg/pymongo/connection.py", line 586, in __find_node
raise AutoReconnect(', '.join(errors))
pymongo.errors.AutoReconnect: could not connect to localhost:27017: [Errno 111] Connection refused
charles#charles-mbp:~/Desktop$ python testmongo.py
Traceback (most recent call last):
File "testmongo.py", line 7, in <module>
conn = Connection()
File "/usr/local/lib/python2.7/dist-packages/pymongo-2.2.1-py2.7-linux-x86_64.egg/pymongo/connection.py", line 290, in __init__
self.__find_node()
File "/usr/local/lib/python2.7/dist-packages/pymongo-2.2.1-py2.7-linux-x86_64.egg/pymongo/connection.py", line 586, in __find_node
raise AutoReconnect(', '.join(errors))
pymongo.errors.AutoReconnect: could not connect to localhost:27017: [Errno 111] Connection refused
charles#charles-mbp:~/Desktop$ python testmongo.py
Traceback (most recent call last):
File "testmongo.py", line 7, in <module>
conn = Connection()
File "/usr/local/lib/python2.7/dist-packages/pymongo-2.2.1-py2.7-linux-x86_64.egg/pymongo/connection.py", line 290, in __init__
self.__find_node()
File "/usr/local/lib/python2.7/dist-packages/pymongo-2.2.1-py2.7-linux-x86_64.egg/pymongo/connection.py", line 586, in __find_node
raise AutoReconnect(', '.join(errors))
pymongo.errors.AutoReconnect: could not connect to localhost:27017: [Errno 111] Connection refused
Some useful information:
The remote server is a VPS running on OpenVZ. I know there are compatibility problems related to count() and iterators, but I (assumingly) fixed them by using the --smallfiles option, as well as limiting my nssize to 100. My firewall on both sides are completely open.
I've also noticed that in the errors, autoreconnect is trying to connect to localhost, even though I specified a different IP address.
You need to pass the IP - conn = Connection(ip)