Develop Reference

Flask-SQLAlchemy Queue Pool limit reached

I have an application where a service makes a POST request to a flask app which then inserts the body of request into a postgresql database into a table as well as inserting data into several other tables.
The problem occurs once a curtain amount of requests are made that the queue pool limit is reach and the app throws an error. i have isolated it down to the extra functions called to do the inserting into other tables. i don't understand what is happening.
Each service is a docker container
172.30.0.17 - - [16/Jun/2022 06:25:07] "GET /api/orgs HTTP/1.1" 500 -
Traceback (most recent call last):
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 2095, in __call__
return self.wsgi_app(environ, start_response)
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 2080, in wsgi_app
response = self.handle_exception(e)
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 2077, in wsgi_app
response = self.full_dispatch_request()
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 1525, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 1523, in full_dispatch_request
rv = self.dispatch_request()
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 1509, in dispatch_request
return self.ensure_sync(self.view_functions[rule.endpoint])(**req.view_args)
File "/service/routes/get/get_org.py", line 17, in base_route
query = [convert_class(item) for item in Organization.query.all()]
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/orm/query.py", line 2768, in all
return self._iter().all()
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/orm/query.py", line 2903, in _iter
result = self.session.execute(
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 1711, in execute
conn = self._connection_for_bind(bind)
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 1552, in _connection_for_bind
return self._transaction._connection_for_bind(
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 747, in _connection_for_bind
conn = bind.connect()
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3234, in connect
return self._connection_cls(self, close_with_result=close_with_result)
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 96, in __init__
else engine.raw_connection()
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3313, in raw_connection
return self._wrap_pool_connect(self.pool.connect, _connection)
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3280, in _wrap_pool_connect
return fn()
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 310, in connect
return _ConnectionFairy._checkout(self)
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 868, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 476, in checkout
rec = pool._do_get()
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 134, in _do_get
raise exc.TimeoutError(
sqlalchemy.exc.TimeoutError: QueuePool limit of size 20 overflow 20 reached, connection timed out, timeout 30.00 (Background on this error at: https://sqlalche.me/e/14/3o7r)
Config
class Config:
# Database
SQLALCHEMY_DATABASE_URI = environ.get("DB_URI")
SQLALCHEMY_ECHO = False
SQLALCHEMY_TRACK_MODIFICATIONS = False
SQLALCHEMY_ENGINE_OPTIONS = {
"pool_size": 1,
"max_overflow": 0,
}
App init
db = SQLAlchemy()
# create flask App
def create_app():
app = Flask(__name__, instance_relative_config=False)
# DB config - postgreSQL
app.config['SQLALCHEMY_DATABASE_URI'] = getenv("DB_URI")
app.config.from_object('config.Config')
# config server with CORS
# cors = CORS(app, resources={r"/*": {"origins": "*"}})
with app.app_context():
db.init_app(app)
from service.routes.post import post_event
return app
app = create_app()
POST request
#app.route('/api/orgs/<org_id>/sites/<site_id>/sessions/<session_id>/events', methods=["POST"])
def edger_post_request(org_id, site_id, session_id):
print("URL params: {}, {}, {}".format(
org_id, site_id, session_id
))
# parse body of post request
try:
request_body = request.json
event_data = json.loads(request_body["data"])
except:
return "error in body of request", 400
print("Body of request: ", request_body)
# check if body of request is valid
errors_body, has_error = validate_input(request_body)
print(errors_body, has_error)
if has_error:
return Response(errors_body, 400)
# insert event
event_dict = dict()
try:
event_dict = {
"id": request_body["id"],
"timestamp": request_body["timestamp"],
"type": request_body["type"],
"event_type": event_data["type"],
"data": request_body["data"],
"org_id": org_id,
"site_id": site_id,
"device_id": request_body["device_id"],
"session_id": request_body["session_id"]
}
if "frame_url" in request_body.keys():
event_dict["frame_url"] = request_body["frame_url"]
else:
event_dict["frame_url"] = None
insert_event = Event(**event_dict)
db.session.add(insert_event)
# commit event to DB
db.session.commit()
# Insert extra tables
insert_threads(org_id, site_id, event_dict, event_data)
db.session.commit()
db.session.close()
return {"result": "success"}, 200
except Exception as e:
print("- - - - - - - - - - - - - - - - -")
db.session.rollback()
print("Error inserting Event", e)
print("- - - - - - - - - - - - - - - - -")
db.session.close()
return {"result": "unsuccessful"}, 409
insert_threads function
def insert_threads(org_id, site_id, body, data):
event_data_insert(body, data),
event_type_insert(data["type"]),
insert_org(org_id),
insert_site(org_id, site_id),
insert_device(org_id, site_id, body['device_id']),
insert_session(org_id, site_id, body['device_id'], body['session_id'], body['timestamp'])
def insert_org(org_id):
org_list = getattr(g, "org_list", None)
# query organization insert if doesn't exist
if org_list is None:
check = [org.id for org in Organization.query.all()]
g.org_list = check
org_list = check
if org_id not in org_list:
insert_org = Organization(id=org_id, name="temp_name")
db.session.add(insert_org)
org_list.append(insert_org.id)
g.org_list = org_list
def insert_site(org_id, site_id):
site_list = getattr(g, "site_list", None)
# query Site insert if doesn't exist
if site_list is None:
check = [site.id for site in Site.query.all()]
g.site_list = check
site_list = check
if site_id not in site_list:
insert_site = Site(id=site_id, org_id=org_id, name="temp_name")
db.session.add(insert_site)
site_list.append(insert_site.id)
g.site_list = site_list
def insert_device(org_id, site_id, device_id):
device_list = getattr(g, "device_list", None)
# query device insert if doesn't exist
if device_list is None:
check = [device.id for device in Device.query.all()]
g.device_list = check
device_list = check
if device_id not in device_list:
insert_device = Device(id=device_id, site_id=site_id, org_id=org_id)
db.session.add(insert_device)
device_list.append(insert_device.id)
g.device_list = device_list
def insert_session(org_id, site_id, device_id, session_id, timestamp):
session_list = getattr(g, "session_list", None)
# query session and insert if it doesn't already exist
if session_list is None:
check = [session.id for session in Session.query.all()]
g.session_list = check
session_list = check
if session_id not in session_list:
insert_session = Session(
id=session_id,
timestamp=timestamp,
org_id=org_id,
site_id=site_id,
device_id=device_id
)
db.session.add(insert_session)
session_list.append(insert_session.id)
g.session_list = session_list
def event_data_insert(event, event_data):
# get white list from global var else None
query_list = getattr(g, "query_list", None)
# if query_list is None get query_list from DB
if query_list is None:
check = [convert_class(item) for item in Event_data_query_list.query.all()]
g.query_list = check
query_list = check
# loop though query_list and insert key/value pair into event_data table
for key in query_list:
print("the key: ", key)
key_path = key["key_name"].split(".")
# check if keys exist in data Object and continue down levels till the value is reached
if key_path[0] in event_data["data"] and key["type"] == event_data["type"]:
print("inside first if")
value = event_data['data']
for sub_key in key_path:
print("inside loop")
if sub_key in value.keys():
print("in subkey" , sub_key)
value = value[sub_key]
# handle if keys and values dont exist
else:
value = None
break
if value is None:
continue
# determine where value is an Integer or a String and insert into correct column
val_int = None
val_str = None
if type(value) == str:
val_str = value
elif type(value) == int:
val_int = value
# add event_data row to session
insert = Event_data(event_id=event["id"], type=event_data["type"], key=key["key_name"], value_int=val_int, value_str=val_str )
print("insert object", insert)
db.session.add(insert)
def event_type_insert(event_type):
event_type_list = getattr(g, "event_type_list", None)
if event_type_list is None:
check = [item.event_type for item in Event_type.query.all()]
g.event_type_list = check
event_type_list = check
if event_type not in event_type_list:
insert = Event_type(event_type=event_type)
db.session.add(insert)
g.event_type_list.append(event_type)

Discord.py and SQL - Comparing between 2 tables in the same database file

So I have two tables, Servers and Strikes. In the Servers table, I have a max strikes value. I want to compare the strikes a user has to the max strikes specified in the Servers table. I tried this:
#commands.command(name='strike', pass_ctx= True)
#commands.has_permissions(manage_messages=True)
async def strike(self, ctx, member : discord.Member):
first_strike = 1
cursor = await self.client.db.cursor()
await cursor.execute(f"SELECT Strikes_Have FROM Strikes WHERE Guild_ID = {ctx.guild.id} AND User_ID = {member.id}")
result = await cursor.fetchone()
if result == None:
cursor = await self.client.db.cursor()
sql = f"INSERT INTO Strikes(Guild_ID, User_ID, Strikes_Have) VALUES({ctx.guild.id}, {member.id}, {first_strike})"
else:
cursor = await self.client.db.cursor()
sql = f"UPDATE Strikes SET Strikes_Have = Strikes_Have + 1 where Guild_ID = {ctx.guild.id} AND User_ID = {member.id}"
if result["Strikes_Have"] == result["MAX_STRIKES"]:
await ctx.message.channel.send(f"{member} has the max amount of stirkes specified by the server")
await cursor.execute(sql)
await self.client.db.commit()
await cursor.close()
await ctx.message.channel.send(f"Striked {member}")`
But got this error:
Traceback (most recent call last):
File "/home/pi/.local/lib/python3.7/site-packages/discord/ext/commands/core.py", line 85, in wrapped
ret = await coro(*args, **kwargs)
File "/home/pi/EndorCore/cogs/Mod.py", line 225, in strike
if result["Strikes_Have"] == result["MAX_STRIKES"]:
IndexError: No item with that key

The error message you're getting currently is being caused by you trying to access a column of data which doesn't exist in the strikes table.
To get the max strikes value, you need to query that table and get the number, then compare the two.
For example:
cursor.execute(f"SELECT Strikes_Have FROM Strikes WHERE Guild_ID = {ctx.guild.id} AND User_ID = {member.id}")
user_result = cursor.fetchone()
cursor.execute(f"SELECT MAX_STRIKES FROM Servers WHERE Guild_ID = {ctx.guild.id}")
server_result = cursor.fetchone()
if user_result["Strikes_Have"] == server_result['MAX_STRIKES']:
print("User has max strikes")

why "weakly-referenced object no longer exists" error occurred?

I'm making telegram bot, which is adding some messages to mysql database and then deleting them. When I use this bot only by myself I don't get any errors, but if I sending messages using second account and trying to do it at the same time, after 5-10 messages I get "weakly-referenced object no longer exists" error. I guess I somehow lose connection to mysql database but I don't understand why. That's the error.
2020-08-10 00:05:48,249 (util.py:68 WorkerThread1) ERROR - TeleBot: "ReferenceError occurred, args=('weakly-referenced object no longer exists',)
Traceback (most recent call last):
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\telebot\util.py", line 62, in run
task(*args, **kwargs)
File "C:/shit stuff/code stuff/projects/dota_tg_bot/tg_bot.py", line 47, in process_quiz_second_question
db.add_answer(user_id, int(message.text), 2)
File "C:\shit stuff\code stuff\projects\dota_tg_bot\database.py", line 129, in add_answer
self.cursor.execute(add_answer_statement, (answer, date, user_id))
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\mysql\connector\cursor.py", line 569, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\mysql\connector\cursor.py", line 484, in _handle_result
self._connection.unread_result = False
ReferenceError: weakly-referenced object no longer exists
"
Traceback (most recent call last):
File "C:/shit stuff/code stuff/projects/dota_tg_bot/tg_bot.py", line 136, in <module>
main()
File "C:/shit stuff/code stuff/projects/dota_tg_bot/tg_bot.py", line 132, in main
bot.polling()
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\telebot\__init__.py", line 427, in polling
self.__threaded_polling(none_stop, interval, timeout)
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\telebot\__init__.py", line 451, in __threaded_polling
self.worker_pool.raise_exceptions()
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\telebot\util.py", line 111, in raise_exceptions
six.reraise(self.exc_info[0], self.exc_info[1], self.exc_info[2])
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\six.py", line 703, in reraise
raise value
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\telebot\util.py", line 62, in run
task(*args, **kwargs)
File "C:/shit stuff/code stuff/projects/dota_tg_bot/tg_bot.py", line 47, in process_quiz_second_question
db.add_answer(user_id, int(message.text), 2)
File "C:\shit stuff\code stuff\projects\dota_tg_bot\database.py", line 129, in add_answer
self.cursor.execute(add_answer_statement, (answer, date, user_id))
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\mysql\connector\cursor.py", line 569, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\mysql\connector\cursor.py", line 484, in _handle_result
self._connection.unread_result = False
ReferenceError: weakly-referenced object no longer exists
Process finished with exit code 1
That's class of mysql handler i use.
import mysql.connector
from config import DB_PASSWORD
from dota_parser import get_winrates, get_quiz_answers
import datetime
class MySQLDatabase:
def __init__(self):
self.connection = None
self.cursor = None
def connect(self):
self.connection = mysql.connector.connect(
host='localhost',
user='root',
password=DB_PASSWORD,
port='3306',
database='dotatelegrambot'
)
self.cursor = self.connection.cursor()
def close(self):
self.cursor.close()
self.connection.close()
def get_hero_answers(self, hero):
self.connect()
get_hero_answers_statement = ('SELECT Answer1, Answer2, Answer3 FROM heroes '
'WHERE Hero = %s')
self.cursor.execute(get_hero_answers_statement, (hero,))
answers = self.cursor.fetchone()
self.close()
return answers
def get_hero_winrates(self, hero):
self.connect()
get_winrate_statement = ('SELECT Winrate1, Winrate2, Winrate3, Winrate4, Winrate5 '
'FROM heroes '
'WHERE Hero = %s')
self.cursor.execute(get_winrate_statement, (hero,))
winrates = self.cursor.fetchone()
self.close()
return winrates
def get_heroes(self):
self.connect()
get_heroes_statement = 'SELECT Hero FROM heroes'
self.cursor.execute(get_heroes_statement)
heroes = self.cursor.fetchall()
self.close()
return heroes
def update_user(self, user_id, hero):
self.connect()
get_user_statement = ('SELECT UserID '
'FROM users '
'WHERE UserID = %s')
self.cursor.execute(get_user_statement, (user_id,))
user = self.cursor.fetchone()
now = datetime.datetime.now()
date = f'{now.year}-{now.month}-{now.day}'
if user:
update_user_statement = ('UPDATE users '
'SET CurrentHero = %s, LastDate = %s '
'WHERE UserID = %s')
self.cursor.execute(update_user_statement, (hero, date, user_id))
self.connection.commit()
else:
add_user_statement = ('INSERT INTO users (UserID, CurrentHero, LastDate) '
'VALUES (%s, %s, %s)')
self.cursor.execute(add_user_statement, (user_id, hero, date))
self.connection.commit()
self.close()
def get_user_hero(self, user_id):
self.connect()
get_user_hero_statement = ('SELECT CurrentHero '
'FROM users '
'WHERE UserID = %s')
self.cursor.execute(get_user_hero_statement, (user_id,))
hero = self.cursor.fetchone()[0]
self.close()
return hero
def add_answer(self, user_id, answer, number):
self.connect()
now = datetime.datetime.now()
date = f'{now.year}-{now.month}-{now.day}'
add_answer_statement = ('UPDATE users '
f'SET Answer{number} = %s, LastDate = %s '
'WHERE UserID = %s')
self.cursor.execute(add_answer_statement, (answer, date, user_id))
self.connection.commit()
self.close()
def get_user_answers(self, user_id):
self.connect()
get_user_answers_statement = ('SELECT Answer1, Answer2, Answer3 FROM users '
'WHERE UserID = %s')
self.cursor.execute(get_user_answers_statement, (user_id,))
answers = self.cursor.fetchone()
self.close()
return answers
def clear_answers(self, user_id):
self.connect()
now = datetime.datetime.now()
date = f'{now.year}-{now.month}-{now.day}'
clear_answers_statement = ('UPDATE users '
'SET CurrentHero = Null, Answer1 = Null, Answer2 = Null, Answer3 = Null, '
'LastDate = %s '
'WHERE UserID = %s')
self.cursor.execute(clear_answers_statement, (date, user_id))
self.connection.commit()
self.close()
def main():
db = MySQLDatabase()
db.connect()
db.close()
if __name__ == '__main__':
main()
Bot code.
import telebot
from telebot import types
import random
from config import TG_TOKEN, DOTA_RANKS
from database import MySQLDatabase
bot = telebot.TeleBot(TG_TOKEN)
db = MySQLDatabase()
#bot.message_handler(commands=['start'])
def welcome(message):
bot.send_message(message.from_user.id, 'Hi, I\'m dotabuff bot.')
bot.send_message(message.from_user.id, 'What do you want to do?', reply_markup=welcome_keyboard())
#bot.message_handler(commands=['quiz'])
def start_quiz(message):
msg = bot.send_message(message.from_user.id, 'Choose hero:', reply_markup=heroes_keyboard())
bot.register_next_step_handler(msg, process_quiz_choose_hero)
def process_quiz_choose_hero(message):
if cancel_check(message):
hero = message.text
user_id = int(message.from_user.id)
db.update_user(user_id, hero)
msg = bot.send_message(message.from_user.id, f'What is starting move speed of {hero}?',
reply_markup=quiz_answers_keyboard(db.get_hero_answers(hero)[0]))
bot.register_next_step_handler(msg, process_quiz_first_question)
def process_quiz_first_question(message):
if cancel_check(message):
user_id = int(message.from_user.id)
hero = db.get_user_hero(user_id)
db.add_answer(user_id, int(message.text), 1)
msg = bot.send_message(message.from_user.id, f'What is starting attack speed of {hero}?',
reply_markup=quiz_answers_keyboard(db.get_hero_answers(hero)[1]))
bot.register_next_step_handler(msg, process_quiz_second_question)
def process_quiz_second_question(message):
if cancel_check(message):
user_id = int(message.from_user.id)
hero = db.get_user_hero(user_id)
db.add_answer(user_id, int(message.text), 2)
msg = bot.send_message(message.from_user.id, f'What is starting attack range of {hero}?',
reply_markup=quiz_answers_keyboard(db.get_hero_answers(hero)[2]))
bot.register_next_step_handler(msg, process_quiz_third_question)
def process_quiz_third_question(message):
if cancel_check(message):
user_id = int(message.from_user.id)
hero = db.get_user_hero(user_id)
db.add_answer(user_id, int(message.text), 3)
correct_ans = 0
for ans_num in range(3):
correct_ans += db.get_hero_answers(hero)[ans_num] == db.get_user_answers(user_id)[ans_num]
if correct_ans != 1:
bot.send_message(user_id, f'Congratulations!!! You have exactly {correct_ans} correct answers.',
reply_markup=welcome_keyboard())
else:
bot.send_message(user_id, f'Congratulations!!! You have exactly {correct_ans} correct answer.',
reply_markup=welcome_keyboard())
db.clear_answers(user_id)
#bot.message_handler(commands=['winrates'])
def start_wr_check(message):
msg = bot.send_message(message.from_user.id, 'Choose hero:', reply_markup=heroes_keyboard())
bot.register_next_step_handler(msg, process_hero_winrate)
def process_hero_winrate(message):
if cancel_check(message):
hero = message.text
reply = ''
heroes_winrates = db.get_hero_winrates(hero)
for i, rank in enumerate(DOTA_RANKS):
line = f'{rank} — *{str(heroes_winrates[i])} %* \n'
reply += line
bot.send_message(message.from_user.id, reply, parse_mode='Markdown', reply_markup=welcome_keyboard())
#bot.message_handler(func=lambda message: message.text == 'Cancel ⛔️')
def menu(message):
bot.send_message(message.from_user.id, 'What do you want to do?', reply_markup=welcome_keyboard())
def welcome_keyboard():
markup = types.ReplyKeyboardMarkup(one_time_keyboard=True, resize_keyboard=True)
markup.add('/winrates', '/quiz')
return markup
def heroes_keyboard():
markup = types.ReplyKeyboardMarkup(one_time_keyboard=True, resize_keyboard=True)
markup.add('Cancel ⛔️')
heroes = db.get_heroes()
for hero in heroes:
markup.add(hero[0])
return markup
def quiz_answers_keyboard(correct_answer):
markup = types.ReplyKeyboardMarkup(one_time_keyboard=True, resize_keyboard=True)
markup.add('Cancel ⛔️')
answer_buttons = [types.KeyboardButton(correct_answer)]
int_ans = int(correct_answer)
for _ in range(2):
rand_answers = [x for x in range(int_ans - 45, int_ans - 4, 5)] +\
[x for x in range(int_ans + 5, int_ans + 46, 5)]
rand_ans = rand_answers.pop(random.randrange(len(rand_answers)))
answer_buttons.append(types.KeyboardButton(rand_ans))
random.shuffle(answer_buttons)
for button in answer_buttons:
markup.add(button)
return markup
def cancel_check(message):
if message.text == 'Cancel ⛔️':
bot.clear_step_handler_by_chat_id(message.from_user.id)
bot.send_message(message.from_user.id, 'What do you want to do?', reply_markup=welcome_keyboard())
return False
return True
def main():
bot.polling()
if __name__ == '__main__':
main()
So every time I need to take or input data in database I open connection and after it I close it. (There is a lot of non problem related stuff use Ctrl+F "db" to follow database methods calls.)
https://imgur.com/a/hSrAE4D
That's the structure of db for better understanding.

I'm not familiar enough with database related python's stuff, but I've seen in your code that every method you defined in MySqlDatabase creates a new instance of the database connection.
Are you sure you don't need to check if there is no connection open - to avoid instantiating a new one?
Here is how I would do:
def connect(self):
self.connection = mysql.connector.connect(
host='localhost',
user='root',
password=DB_PASSWORD,
port='3306',
database='dotatelegrambot'
)
def getCursor(self)
if not self.connection:
self.connect()
return self.connection.cursor()
def get_user_hero(self, user_id):
cursor = self.getCursor()
get_user_hero_statement = ('SELECT CurrentHero '
'FROM users '
'WHERE UserID = %s')
cursor.execute(get_user_hero_statement, (user_id,))
hero = cursor.fetchone()[0]
cursor.close()
return hero

aioMySQL not connecting with python

I know this is somewhat of a duplicate, but I've looked in every nook and cranny of the internet and I can't seem to find the solution to this. Basically, I'm trying to connect to a mySQL database via aioMySQL. My Script is far too long to put in, but it was able to connect to an already setup mySQL/phpMyAdmin just fine. When I run it trying to connect to the new db, I get
Task exception was never retrieved
future: <Task finished name='Task-1' coro=<database.createPool() done, defined at /home/blackcoffee/Desktop/Code/She11Sh0ck/dbhandler.py:13> exception=OperationalError(2003, "Can't connect to MySQL server on 'X.X.X.X'")>
Traceback (most recent call last):
File "/home/blackcoffee/.local/lib/python3.8/site-packages/aiomysql/connection.py", line 486, in _connect
self._reader, self._writer = await \
File "/usr/lib/python3.8/asyncio/tasks.py", line 455, in wait_for
return await fut
File "/usr/lib/python3.8/asyncio/streams.py", line 52, in open_connection
transport, _ = await loop.create_connection(
File "/usr/lib/python3.8/asyncio/base_events.py", line 1017, in create_connection
raise exceptions[0]
File "/usr/lib/python3.8/asyncio/base_events.py", line 1002, in create_connection
sock = await self._connect_sock(
File "/usr/lib/python3.8/asyncio/base_events.py", line 916, in _connect_sock
await self.sock_connect(sock, address)
File "/usr/lib/python3.8/asyncio/selector_events.py", line 485, in sock_connect
return await fut
File "/usr/lib/python3.8/asyncio/selector_events.py", line 517, in _sock_connect_cb
raise OSError(err, f'Connect call failed {address}')
ConnectionRefusedError: [Errno 111] Connect call failed ('X.X.X.X', 3306)
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/blackcoffee/Desktop/Code/She11Sh0ck/dbhandler.py", line 15, in createPool
self.pool = await aiomysql.create_pool(
File "/home/blackcoffee/.local/lib/python3.8/site-packages/aiomysql/pool.py", line 29, in _create_pool
await pool._fill_free_pool(False)
File "/home/blackcoffee/.local/lib/python3.8/site-packages/aiomysql/pool.py", line 167, in _fill_free_pool
conn = await connect(echo=self._echo, loop=self._loop,
File "/home/blackcoffee/.local/lib/python3.8/site-packages/aiomysql/connection.py", line 75, in _connect
await conn._connect()
File "/home/blackcoffee/.local/lib/python3.8/site-packages/aiomysql/connection.py", line 521, in _connect
raise OperationalError(2003,
pymysql.err.OperationalError: (2003, "Can't connect to MySQL server on 'X.X.X.X'")
Looking at some details on my server,
mysqladmin Ver 8.42 Distrib 5.7.29, for Linux on x86_64
Copyright (c) 2000, 2020, Oracle and/or its affiliates. All rights reserved.
Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.
Server version 5.7.29-0ubuntu0.18.04.1
Protocol version 10
Connection Localhost via UNIX socket
UNIX socket /var/run/mysqld/mysqld.sock
Uptime: 5 min 26 sec
Threads: 1 Questions: 4 Slow queries: 0 Opens: 105 Flush tables: 1 Open tables: 98 Queries per second avg: 0.012
Lastly, I know my root password is correct because I can log in to phpMyAdmin with it with no problems. Any help would be greatly appreciated.
Progress Update: I got it to connect by changing the bind-address to 0.0.0.0, but now it's giving me another error. Here is the code to handle the db as well as the error.
import asyncio
import aiomysql
from creds import getdb
# Connect to server
class database:
def __init__(self):
print("SQL DB STARTED")
async def createPool(self, loop):
dbcreds = getdb()
self.pool = await aiomysql.create_pool(
host=dbcreds["host"],
port=dbcreds["port"],
user=dbcreds["user"],
password=dbcreds["password"],
db=dbcreds["db"],
loop=loop,
)
async def getUserCTFID(self, discordID, guildID):
sql = "SELECT activectf FROM members where `uuid` = %d and `guildid` = %d" % (
int(discordID),
int(guildID),
)
async with self.pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
(r) = await cur.fetchone()
return r[0]
# self.pool.close()
# await self.pool.wait_closed()
async def getCTFID(self, name, guildID):
sql = "SELECT ctfid FROM ctfs where `name` = '{}' and `guildid` = '{}'".format(
str(name), int(guildID),
)
async with self.pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
(r) = await cur.fetchone()
return r[0]
async def getCTFName(self, CTFID):
sql = "SELECT name FROM ctfs where `ctfid` = %d" % (int(CTFID))
async with self.pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
(r) = await cur.fetchone()
return r[0]
# self.pool.close()
# wait self.pool.wait_closed()
async def getCTFQuestions(self, CTFID):
sql = "SELECT name,Solved FROM ctfQuestions where `ctfid` = %d" % (int(CTFID))
async with self.pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
(r) = await cur.fetchall()
return r
# self.pool.close()
# await self.pool.wait_closed()
async def getValidCTFIDs(self, DiscordID, GuildID):
sql = "SELECT ctfs.ctfid,ctfs.name FROM members INNER JOIN ctfs ON ctfs.guildid=members.guildid WHERE ctfs.guildid = members.guildid and members.uuid = {} and members.guildid = {}".format(
int(DiscordID), int(GuildID)
)
async with self.pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
(r) = await cur.fetchall()
return r
# self.pool.close()
# await self.pool.wait_closed()
async def updateCTF(self, DiscordID, GuildID, CTFID):
sql = "UPDATE `members` SET `activectf`={} WHERE uuid={} and guildid={}".format(
int(CTFID), int(DiscordID), int(GuildID)
)
async with self.pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
await conn.commit()
# self.pool.close()
# await self.pool.wait_closed()
async def createCTF(self, ctfName, guildID):
print(ctfName)
sql = "INSERT INTO ctfs (name, guildid) VALUES ('{}','{}')".format(
str(ctfName), int(guildID)
)
async with self.pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
await conn.commit()
async def deleteCTF(self, ctfName, guildID):
print("Goodbye {}".format(ctfName))
sql = "DELETE FROM `ctfs` WHERE name = '{}' and guildid = '{}'".format(
str(ctfName), int(guildID)
)
async with self.pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
await conn.commit()
# self.pool.close()
# await self.pool.wait_closed()
async def getGuildByID(self, guildid):
sql = "SELECT guildid, guildname from guilds where guildid={}".format(
int(guildid)
)
async with self.pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
return await cur.fetchone()
# self.pool.close()
# await self.pool.wait_closed()
async def getMember(self, uuid, guildid):
sql = "SELECT id from members where uuid = {} and guildid={}".format(
int(uuid), int(guildid)
)
async with self.pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
return await cur.fetchone()
# self.pool.close()
# await self.pool.wait_closed()
async def addMember(self, uuid, guildid):
sql = "INSERT INTO members (uuid,guildid, activectf) VALUES ('{}','{}','{}')".format(
int(uuid), int(guildid), 0
)
async with self.pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
await conn.commit()
# self.pool.close()
# await self.pool.wait_closed()
async def addGuild(self, guildid, guildname):
sql = "INSERT INTO guilds (guildid, guildname) VALUES ('{}','{}')".format(
int(guildid), str(guildname)
)
print(sql)
async with self.pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
await conn.commit()
# self.pool.close()
# await self.pool.wait_closed()
async def addQuestion(self, questionName, CTFID):
sql = "INSERT INTO ctfQuestions (name, ctfid, Solved) VALUES ('{}','{}', '{}')".format(
str(questionName), int(CTFID), 0
)
async with self.pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
await conn.commit()
# self.pool.close()
# await self.pool.wait_closed()
async def updateQuestionState(self, questionName, CTFID, state):
sql = "UPDATE `ctfQuestions` SET `Solved`='{}' WHERE name='{}' and CTFID='{}'".format(
int(state), str(questionName), int(CTFID)
)
async with self.pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
await conn.commit()
# self.pool.close()
# await self.pool.wait_closed()
async def setSolved(self, questionName, CTFID):
await self.updateQuestionState(questionName, CTFID, 1)
async def setUnsolved(self, questionName, CTFID):
await self.updateQuestionState(questionName, CTFID, 0)
async def delQuestion(self, questionName, CTFID):
sql = "DELETE FROM `ctfQuestions` WHERE name='{}' and CTFID='{}' ".format(
str(questionName), int(CTFID)
)
async with self.pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(sql)
await conn.commit()
# self.pool.close()
# await self.pool.wait_closed()
An open stream object is being garbage collected; call "stream.close()" explicitly.
Exception ignored in: <coroutine object Connection._get_server_information at 0x7fedfa7be6c0>
Traceback (most recent call last):
File "/home/blackcoffee/.local/lib/python3.8/site-packages/aiomysql/connection.py", line 989, in _get_server_information
packet = await self._read_packet()
File "/home/blackcoffee/.local/lib/python3.8/site-packages/aiomysql/connection.py", line 561, in _read_packet
packet_header = await self._read_bytes(4)
File "/home/blackcoffee/.local/lib/python3.8/site-packages/aiomysql/connection.py", line 599, in _read_bytes
except asyncio.streams.IncompleteReadError as e:
AttributeError: module 'asyncio.streams' has no attribute 'IncompleteReadError'
Task was destroyed but it is pending!
task: <Task pending name='Task-1' coro=<database.createPool() running at /home/blackcoffee/Desktop/Code/She11Sh0ck/dbhandler.py:15> wait_for=<Future pending cb=[<TaskWakeupMethWrapper object at 0x7fedf9624ee0>()]>>
To be honest, I have no idea what to make of this error
SELECT ##thread_handling; gives:
+---------------------------+
| ##thread_handling |
+---------------------------+
| one-thread-per-connection |
+---------------------------+
1 row in set (0.00 sec)

I feel like such an idiot. After days and days of searching, the solution was to allow a connection from my own ip.
GRANT ALL ON *.* to <user>#<my_ip> IDENTIFIED BY <my_password>;

passing psycopg2 cursor to tweepy on_status() method

I'm trying to pass a psycopg2 cursor to a tweepy stream.
The connection pool and cursors are configured in a seperate file. Only the cursor is passed as an argument to the main pipeline function in another file called get_tweet_topic.py. I need the cursor in the on_status() method because I have a query there that needs it for execution.
I cannot figure out how to pass the cursor onto the on_status() method in the MyStreamListener() class.
the error I get is:
2020-03-05T22:16:24.856945+00:00 app[worker.1]: self._target(*self._args, **self._kwargs)
2020-03-05T22:16:24.856945+00:00 app[worker.1]: File "/app/get_tweet_topic.py", line 81, in guess_topic_pipeline
2020-03-05T22:16:24.856946+00:00 app[worker.1]: status_streams.streaming_pipeline(api, cursor)
2020-03-05T22:16:24.856947+00:00 app[worker.1]: File "/app/status_streams.py", line 100, in streaming_pipeline
2020-03-05T22:16:24.856947+00:00 app[worker.1]: general_stream(api, cursor)
2020-03-05T22:16:24.856948+00:00 app[worker.1]: File "/app/status_streams.py", line 86, in general_stream
2020-03-05T22:16:24.856948+00:00 app[worker.1]: myStreamListener = MyStreamListener()
2020-03-05T22:16:24.856948+00:00 app[worker.1]: TypeError: __init__() missing 1 required positional argument: 'cursor'
Code:
status_streams.py:
import tweepy
import os
import db_queries
import follow
#define class for the stream listener
class MyStreamListener(tweepy.StreamListener):
def __init__(self, cursor):
super().__init__()
self.cursor = cursor
#set counter to only get 1200 tweets
self.counter = 0
self.max = 1200
#get tweets
def on_status(self, status):
if not status.retweeted:
status_dict = {'created_at': status.created_at.strftime('%y-%m-&d %H:%M'),
'source_stream': 'general stream',
'status_id': status.id_str,
'user_id': status.user.id_str,
'screen_name': status.user.name,
'tweet_text': status.text,
'num_likes': status.favorite_count,
'num_retweets': status.retweet_count}
created_at = status_dict['created_at']
source_stream = status_dict['source_stream']
status_id = status_dict['status_id']
user_id = status_dict['user_id']
screen_name = status_dict['screen_name']
tweet_text = status_dict['tweet_text']
num_likes = status_dict['num_likes']
num_retweets = status_dict['num_retweets']
db_queries.insert_raw_tweets_table(cursor, created_at, source_stream, status_id, user_id, screen_name, tweet_text, num_likes, num_retweets)
self.counter +=1
if self.counter == self.max:
return False
#get tweets from list of followers
def following_stream(api, cursor, user_name):
try:
for status in tweepy.Cursor(api.user_timeline, tweet_mode='extended', include_rts=False, screen_name=user_name).items(1):
#ignore retweets
if not status.retweeted:
status_dict = {'created_at': status.created_at.strftime('%y-%m-%d %H:%M'),
'source_stream': 'following stream',
'status_id': status.id_str,
'user_id': status.user.id_str,
'screen_name': status.user.name,
'tweet_text':status.full_text,
'num_likes':status.favorite_count,
'num_retweets':status.retweet_count}
created_at = status_dict['created_at']
source_stream = status_dict['source_stream']
status_id = status_dict['status_id']
user_id = status_dict['user_id']
screen_name = status_dict['screen_name']
tweet_text = status_dict['tweet_text']
num_likes = status_dict['num_likes']
num_retweets = status_dict['num_retweets']
db_queries.insert_raw_tweets_table(cursor, created_at, source_stream, status_id, user_id, screen_name, tweet_text, num_likes, num_retweets)
#function that controls both streams
def streaming_pipeline(api, cursor):
#get list of all users that are currently followed
#iterate through the following_list and grab the single latest tweet
following_list = follow.get_following(api)
for user in following_list:
f_stream = following_stream(api, cursor, user)
#stream class is used here
myStreamListener = MyStreamListener()
stream = tweepy.Stream(auth=api.auth, listener=myStreamListener(cursor=self.cursor))
stream.filter(languages=['en'], track=['the'])
cursor.close()
relevant section of get_tweet_topic.py:
def guess_topic_pipeline(api, conn, model, corpus, classifier):
while True:
cursor = conn.cursor()
db_queries.create_temp_tweets_table(cursor)
conn.commit()
#use pipeline to grab tweets off twitter
print('Retrieving statuses from streams...')
status_streams.streaming_pipeline(api, cursor)
print('Done retrieving...')
relevant portion of connection pooling code:
#get connection from pool, pass cursor as an argument, start topic extration thread
topic_conn = conn_pool.getconn()
topic_extraction_thread = Thread(target=get_tweet_topic.guess_topic_pipeline, kwargs={'api':api, 'conn': topic_conn, 'model': lda_model, 'corpus': lda_id2word, 'classifier': lda_huber_classifier})
topic_extraction_thread.start()
#return connection when done
conn_pool.putconn(topic_conn)
insert_raw_tweets_table() function with actual query:
def insert_raw_tweets_table(cursor, createdAt, sourceStream, statusID, userID, screenName, tweetText, numLikes, numRetweets):
cursor.execute('INSERT INTO tempTweets(createdAt, sourceStream, statusID, userID, screenName, tweetText) VALUES(%s, %s, %s, %s, %s, %s, %s, %s)', (createdAt, sourceStream, statusID, userID, screenName, tweetText, numLikes, numRetweets))

#MauriceMeyer answered the question in the comments but here is the working code for clarity.
I forgot to reference the cursor as self.cursor within the class and I forgot to pass the cursor as an argument when creating an instance of the class. I was passing the cursor as an argument after creating the instance, which is not correct.
Correct code:
class MyStreamListener(tweepy.StreamListener):
def __init__(self, cursor):
super().__init__()
self.cursor = cursor
#set counter to only get 1200 tweets
self.counter = 0
self.max = 1200
#get tweets
def on_status(self, status):
if not status.retweeted:
status_dict = {'created_at' : status.created_at.strftime('%y-%m-&d %H:%M'),
'source_stream' : 'general stream',
'status_id' : status.id_str,
'user_id' : status.user.id_str,
'screen_name' : status.user.name,
'tweet_text' : status.text,
'num_likes' : status.favorite_count,
'num_retweets' : status.retweet_count}
created_at = status_dict['created_at']
source_stream = status_dict['source_stream']
status_id = status_dict['status_id']
user_id = status_dict['user_id']
screen_name = status_dict['screen_name']
tweet_text = status_dict['tweet_text']
num_likes = status_dict['num_likes']
num_retweets = status_dict['num_retweets']
#▼ reference self.cursor here
db_queries.insert_raw_tweets_table(self.cursor, created_at, source_stream, status_id, user_id, screen_name, tweet_text, num_likes, num_retweets)
self.counter +=1
if self.counter == self.max:
return False
#stream class is used here ▼ reference cursor here
myStreamListener = MyStreamListener(cursor)
#▼ removed reference to cursor here
stream = tweepy.Stream(auth=api.auth, listener=myStreamListener)
stream.filter(languages=['en'], track=['the'])