I'm making telegram bot, which is adding some messages to mysql database and then deleting them. When I use this bot only by myself I don't get any errors, but if I sending messages using second account and trying to do it at the same time, after 5-10 messages I get "weakly-referenced object no longer exists" error. I guess I somehow lose connection to mysql database but I don't understand why. That's the error.
2020-08-10 00:05:48,249 (util.py:68 WorkerThread1) ERROR - TeleBot: "ReferenceError occurred, args=('weakly-referenced object no longer exists',)
Traceback (most recent call last):
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\telebot\util.py", line 62, in run
task(*args, **kwargs)
File "C:/shit stuff/code stuff/projects/dota_tg_bot/tg_bot.py", line 47, in process_quiz_second_question
db.add_answer(user_id, int(message.text), 2)
File "C:\shit stuff\code stuff\projects\dota_tg_bot\database.py", line 129, in add_answer
self.cursor.execute(add_answer_statement, (answer, date, user_id))
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\mysql\connector\cursor.py", line 569, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\mysql\connector\cursor.py", line 484, in _handle_result
self._connection.unread_result = False
ReferenceError: weakly-referenced object no longer exists
"
Traceback (most recent call last):
File "C:/shit stuff/code stuff/projects/dota_tg_bot/tg_bot.py", line 136, in <module>
main()
File "C:/shit stuff/code stuff/projects/dota_tg_bot/tg_bot.py", line 132, in main
bot.polling()
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\telebot\__init__.py", line 427, in polling
self.__threaded_polling(none_stop, interval, timeout)
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\telebot\__init__.py", line 451, in __threaded_polling
self.worker_pool.raise_exceptions()
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\telebot\util.py", line 111, in raise_exceptions
six.reraise(self.exc_info[0], self.exc_info[1], self.exc_info[2])
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\six.py", line 703, in reraise
raise value
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\telebot\util.py", line 62, in run
task(*args, **kwargs)
File "C:/shit stuff/code stuff/projects/dota_tg_bot/tg_bot.py", line 47, in process_quiz_second_question
db.add_answer(user_id, int(message.text), 2)
File "C:\shit stuff\code stuff\projects\dota_tg_bot\database.py", line 129, in add_answer
self.cursor.execute(add_answer_statement, (answer, date, user_id))
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\mysql\connector\cursor.py", line 569, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\mysql\connector\cursor.py", line 484, in _handle_result
self._connection.unread_result = False
ReferenceError: weakly-referenced object no longer exists
Process finished with exit code 1
That's class of mysql handler i use.
import mysql.connector
from config import DB_PASSWORD
from dota_parser import get_winrates, get_quiz_answers
import datetime
class MySQLDatabase:
def __init__(self):
self.connection = None
self.cursor = None
def connect(self):
self.connection = mysql.connector.connect(
host='localhost',
user='root',
password=DB_PASSWORD,
port='3306',
database='dotatelegrambot'
)
self.cursor = self.connection.cursor()
def close(self):
self.cursor.close()
self.connection.close()
def get_hero_answers(self, hero):
self.connect()
get_hero_answers_statement = ('SELECT Answer1, Answer2, Answer3 FROM heroes '
'WHERE Hero = %s')
self.cursor.execute(get_hero_answers_statement, (hero,))
answers = self.cursor.fetchone()
self.close()
return answers
def get_hero_winrates(self, hero):
self.connect()
get_winrate_statement = ('SELECT Winrate1, Winrate2, Winrate3, Winrate4, Winrate5 '
'FROM heroes '
'WHERE Hero = %s')
self.cursor.execute(get_winrate_statement, (hero,))
winrates = self.cursor.fetchone()
self.close()
return winrates
def get_heroes(self):
self.connect()
get_heroes_statement = 'SELECT Hero FROM heroes'
self.cursor.execute(get_heroes_statement)
heroes = self.cursor.fetchall()
self.close()
return heroes
def update_user(self, user_id, hero):
self.connect()
get_user_statement = ('SELECT UserID '
'FROM users '
'WHERE UserID = %s')
self.cursor.execute(get_user_statement, (user_id,))
user = self.cursor.fetchone()
now = datetime.datetime.now()
date = f'{now.year}-{now.month}-{now.day}'
if user:
update_user_statement = ('UPDATE users '
'SET CurrentHero = %s, LastDate = %s '
'WHERE UserID = %s')
self.cursor.execute(update_user_statement, (hero, date, user_id))
self.connection.commit()
else:
add_user_statement = ('INSERT INTO users (UserID, CurrentHero, LastDate) '
'VALUES (%s, %s, %s)')
self.cursor.execute(add_user_statement, (user_id, hero, date))
self.connection.commit()
self.close()
def get_user_hero(self, user_id):
self.connect()
get_user_hero_statement = ('SELECT CurrentHero '
'FROM users '
'WHERE UserID = %s')
self.cursor.execute(get_user_hero_statement, (user_id,))
hero = self.cursor.fetchone()[0]
self.close()
return hero
def add_answer(self, user_id, answer, number):
self.connect()
now = datetime.datetime.now()
date = f'{now.year}-{now.month}-{now.day}'
add_answer_statement = ('UPDATE users '
f'SET Answer{number} = %s, LastDate = %s '
'WHERE UserID = %s')
self.cursor.execute(add_answer_statement, (answer, date, user_id))
self.connection.commit()
self.close()
def get_user_answers(self, user_id):
self.connect()
get_user_answers_statement = ('SELECT Answer1, Answer2, Answer3 FROM users '
'WHERE UserID = %s')
self.cursor.execute(get_user_answers_statement, (user_id,))
answers = self.cursor.fetchone()
self.close()
return answers
def clear_answers(self, user_id):
self.connect()
now = datetime.datetime.now()
date = f'{now.year}-{now.month}-{now.day}'
clear_answers_statement = ('UPDATE users '
'SET CurrentHero = Null, Answer1 = Null, Answer2 = Null, Answer3 = Null, '
'LastDate = %s '
'WHERE UserID = %s')
self.cursor.execute(clear_answers_statement, (date, user_id))
self.connection.commit()
self.close()
def main():
db = MySQLDatabase()
db.connect()
db.close()
if __name__ == '__main__':
main()
Bot code.
import telebot
from telebot import types
import random
from config import TG_TOKEN, DOTA_RANKS
from database import MySQLDatabase
bot = telebot.TeleBot(TG_TOKEN)
db = MySQLDatabase()
#bot.message_handler(commands=['start'])
def welcome(message):
bot.send_message(message.from_user.id, 'Hi, I\'m dotabuff bot.')
bot.send_message(message.from_user.id, 'What do you want to do?', reply_markup=welcome_keyboard())
#bot.message_handler(commands=['quiz'])
def start_quiz(message):
msg = bot.send_message(message.from_user.id, 'Choose hero:', reply_markup=heroes_keyboard())
bot.register_next_step_handler(msg, process_quiz_choose_hero)
def process_quiz_choose_hero(message):
if cancel_check(message):
hero = message.text
user_id = int(message.from_user.id)
db.update_user(user_id, hero)
msg = bot.send_message(message.from_user.id, f'What is starting move speed of {hero}?',
reply_markup=quiz_answers_keyboard(db.get_hero_answers(hero)[0]))
bot.register_next_step_handler(msg, process_quiz_first_question)
def process_quiz_first_question(message):
if cancel_check(message):
user_id = int(message.from_user.id)
hero = db.get_user_hero(user_id)
db.add_answer(user_id, int(message.text), 1)
msg = bot.send_message(message.from_user.id, f'What is starting attack speed of {hero}?',
reply_markup=quiz_answers_keyboard(db.get_hero_answers(hero)[1]))
bot.register_next_step_handler(msg, process_quiz_second_question)
def process_quiz_second_question(message):
if cancel_check(message):
user_id = int(message.from_user.id)
hero = db.get_user_hero(user_id)
db.add_answer(user_id, int(message.text), 2)
msg = bot.send_message(message.from_user.id, f'What is starting attack range of {hero}?',
reply_markup=quiz_answers_keyboard(db.get_hero_answers(hero)[2]))
bot.register_next_step_handler(msg, process_quiz_third_question)
def process_quiz_third_question(message):
if cancel_check(message):
user_id = int(message.from_user.id)
hero = db.get_user_hero(user_id)
db.add_answer(user_id, int(message.text), 3)
correct_ans = 0
for ans_num in range(3):
correct_ans += db.get_hero_answers(hero)[ans_num] == db.get_user_answers(user_id)[ans_num]
if correct_ans != 1:
bot.send_message(user_id, f'Congratulations!!! You have exactly {correct_ans} correct answers.',
reply_markup=welcome_keyboard())
else:
bot.send_message(user_id, f'Congratulations!!! You have exactly {correct_ans} correct answer.',
reply_markup=welcome_keyboard())
db.clear_answers(user_id)
#bot.message_handler(commands=['winrates'])
def start_wr_check(message):
msg = bot.send_message(message.from_user.id, 'Choose hero:', reply_markup=heroes_keyboard())
bot.register_next_step_handler(msg, process_hero_winrate)
def process_hero_winrate(message):
if cancel_check(message):
hero = message.text
reply = ''
heroes_winrates = db.get_hero_winrates(hero)
for i, rank in enumerate(DOTA_RANKS):
line = f'{rank} — *{str(heroes_winrates[i])} %* \n'
reply += line
bot.send_message(message.from_user.id, reply, parse_mode='Markdown', reply_markup=welcome_keyboard())
#bot.message_handler(func=lambda message: message.text == 'Cancel ⛔️')
def menu(message):
bot.send_message(message.from_user.id, 'What do you want to do?', reply_markup=welcome_keyboard())
def welcome_keyboard():
markup = types.ReplyKeyboardMarkup(one_time_keyboard=True, resize_keyboard=True)
markup.add('/winrates', '/quiz')
return markup
def heroes_keyboard():
markup = types.ReplyKeyboardMarkup(one_time_keyboard=True, resize_keyboard=True)
markup.add('Cancel ⛔️')
heroes = db.get_heroes()
for hero in heroes:
markup.add(hero[0])
return markup
def quiz_answers_keyboard(correct_answer):
markup = types.ReplyKeyboardMarkup(one_time_keyboard=True, resize_keyboard=True)
markup.add('Cancel ⛔️')
answer_buttons = [types.KeyboardButton(correct_answer)]
int_ans = int(correct_answer)
for _ in range(2):
rand_answers = [x for x in range(int_ans - 45, int_ans - 4, 5)] +\
[x for x in range(int_ans + 5, int_ans + 46, 5)]
rand_ans = rand_answers.pop(random.randrange(len(rand_answers)))
answer_buttons.append(types.KeyboardButton(rand_ans))
random.shuffle(answer_buttons)
for button in answer_buttons:
markup.add(button)
return markup
def cancel_check(message):
if message.text == 'Cancel ⛔️':
bot.clear_step_handler_by_chat_id(message.from_user.id)
bot.send_message(message.from_user.id, 'What do you want to do?', reply_markup=welcome_keyboard())
return False
return True
def main():
bot.polling()
if __name__ == '__main__':
main()
So every time I need to take or input data in database I open connection and after it I close it. (There is a lot of non problem related stuff use Ctrl+F "db" to follow database methods calls.)
https://imgur.com/a/hSrAE4D
That's the structure of db for better understanding.
I'm not familiar enough with database related python's stuff, but I've seen in your code that every method you defined in MySqlDatabase creates a new instance of the database connection.
Are you sure you don't need to check if there is no connection open - to avoid instantiating a new one?
Here is how I would do:
def connect(self):
self.connection = mysql.connector.connect(
host='localhost',
user='root',
password=DB_PASSWORD,
port='3306',
database='dotatelegrambot'
)
def getCursor(self)
if not self.connection:
self.connect()
return self.connection.cursor()
def get_user_hero(self, user_id):
cursor = self.getCursor()
get_user_hero_statement = ('SELECT CurrentHero '
'FROM users '
'WHERE UserID = %s')
cursor.execute(get_user_hero_statement, (user_id,))
hero = cursor.fetchone()[0]
cursor.close()
return hero
Related
I have an application where a service makes a POST request to a flask app which then inserts the body of request into a postgresql database into a table as well as inserting data into several other tables.
The problem occurs once a curtain amount of requests are made that the queue pool limit is reach and the app throws an error. i have isolated it down to the extra functions called to do the inserting into other tables. i don't understand what is happening.
Each service is a docker container
172.30.0.17 - - [16/Jun/2022 06:25:07] "GET /api/orgs HTTP/1.1" 500 -
Traceback (most recent call last):
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 2095, in __call__
return self.wsgi_app(environ, start_response)
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 2080, in wsgi_app
response = self.handle_exception(e)
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 2077, in wsgi_app
response = self.full_dispatch_request()
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 1525, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 1523, in full_dispatch_request
rv = self.dispatch_request()
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 1509, in dispatch_request
return self.ensure_sync(self.view_functions[rule.endpoint])(**req.view_args)
File "/service/routes/get/get_org.py", line 17, in base_route
query = [convert_class(item) for item in Organization.query.all()]
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/orm/query.py", line 2768, in all
return self._iter().all()
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/orm/query.py", line 2903, in _iter
result = self.session.execute(
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 1711, in execute
conn = self._connection_for_bind(bind)
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 1552, in _connection_for_bind
return self._transaction._connection_for_bind(
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 747, in _connection_for_bind
conn = bind.connect()
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3234, in connect
return self._connection_cls(self, close_with_result=close_with_result)
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 96, in __init__
else engine.raw_connection()
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3313, in raw_connection
return self._wrap_pool_connect(self.pool.connect, _connection)
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3280, in _wrap_pool_connect
return fn()
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 310, in connect
return _ConnectionFairy._checkout(self)
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 868, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 476, in checkout
rec = pool._do_get()
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 134, in _do_get
raise exc.TimeoutError(
sqlalchemy.exc.TimeoutError: QueuePool limit of size 20 overflow 20 reached, connection timed out, timeout 30.00 (Background on this error at: https://sqlalche.me/e/14/3o7r)
Config
class Config:
# Database
SQLALCHEMY_DATABASE_URI = environ.get("DB_URI")
SQLALCHEMY_ECHO = False
SQLALCHEMY_TRACK_MODIFICATIONS = False
SQLALCHEMY_ENGINE_OPTIONS = {
"pool_size": 1,
"max_overflow": 0,
}
App init
db = SQLAlchemy()
# create flask App
def create_app():
app = Flask(__name__, instance_relative_config=False)
# DB config - postgreSQL
app.config['SQLALCHEMY_DATABASE_URI'] = getenv("DB_URI")
app.config.from_object('config.Config')
# config server with CORS
# cors = CORS(app, resources={r"/*": {"origins": "*"}})
with app.app_context():
db.init_app(app)
from service.routes.post import post_event
return app
app = create_app()
POST request
#app.route('/api/orgs/<org_id>/sites/<site_id>/sessions/<session_id>/events', methods=["POST"])
def edger_post_request(org_id, site_id, session_id):
print("URL params: {}, {}, {}".format(
org_id, site_id, session_id
))
# parse body of post request
try:
request_body = request.json
event_data = json.loads(request_body["data"])
except:
return "error in body of request", 400
print("Body of request: ", request_body)
# check if body of request is valid
errors_body, has_error = validate_input(request_body)
print(errors_body, has_error)
if has_error:
return Response(errors_body, 400)
# insert event
event_dict = dict()
try:
event_dict = {
"id": request_body["id"],
"timestamp": request_body["timestamp"],
"type": request_body["type"],
"event_type": event_data["type"],
"data": request_body["data"],
"org_id": org_id,
"site_id": site_id,
"device_id": request_body["device_id"],
"session_id": request_body["session_id"]
}
if "frame_url" in request_body.keys():
event_dict["frame_url"] = request_body["frame_url"]
else:
event_dict["frame_url"] = None
insert_event = Event(**event_dict)
db.session.add(insert_event)
# commit event to DB
db.session.commit()
# Insert extra tables
insert_threads(org_id, site_id, event_dict, event_data)
db.session.commit()
db.session.close()
return {"result": "success"}, 200
except Exception as e:
print("- - - - - - - - - - - - - - - - -")
db.session.rollback()
print("Error inserting Event", e)
print("- - - - - - - - - - - - - - - - -")
db.session.close()
return {"result": "unsuccessful"}, 409
insert_threads function
def insert_threads(org_id, site_id, body, data):
event_data_insert(body, data),
event_type_insert(data["type"]),
insert_org(org_id),
insert_site(org_id, site_id),
insert_device(org_id, site_id, body['device_id']),
insert_session(org_id, site_id, body['device_id'], body['session_id'], body['timestamp'])
def insert_org(org_id):
org_list = getattr(g, "org_list", None)
# query organization insert if doesn't exist
if org_list is None:
check = [org.id for org in Organization.query.all()]
g.org_list = check
org_list = check
if org_id not in org_list:
insert_org = Organization(id=org_id, name="temp_name")
db.session.add(insert_org)
org_list.append(insert_org.id)
g.org_list = org_list
def insert_site(org_id, site_id):
site_list = getattr(g, "site_list", None)
# query Site insert if doesn't exist
if site_list is None:
check = [site.id for site in Site.query.all()]
g.site_list = check
site_list = check
if site_id not in site_list:
insert_site = Site(id=site_id, org_id=org_id, name="temp_name")
db.session.add(insert_site)
site_list.append(insert_site.id)
g.site_list = site_list
def insert_device(org_id, site_id, device_id):
device_list = getattr(g, "device_list", None)
# query device insert if doesn't exist
if device_list is None:
check = [device.id for device in Device.query.all()]
g.device_list = check
device_list = check
if device_id not in device_list:
insert_device = Device(id=device_id, site_id=site_id, org_id=org_id)
db.session.add(insert_device)
device_list.append(insert_device.id)
g.device_list = device_list
def insert_session(org_id, site_id, device_id, session_id, timestamp):
session_list = getattr(g, "session_list", None)
# query session and insert if it doesn't already exist
if session_list is None:
check = [session.id for session in Session.query.all()]
g.session_list = check
session_list = check
if session_id not in session_list:
insert_session = Session(
id=session_id,
timestamp=timestamp,
org_id=org_id,
site_id=site_id,
device_id=device_id
)
db.session.add(insert_session)
session_list.append(insert_session.id)
g.session_list = session_list
def event_data_insert(event, event_data):
# get white list from global var else None
query_list = getattr(g, "query_list", None)
# if query_list is None get query_list from DB
if query_list is None:
check = [convert_class(item) for item in Event_data_query_list.query.all()]
g.query_list = check
query_list = check
# loop though query_list and insert key/value pair into event_data table
for key in query_list:
print("the key: ", key)
key_path = key["key_name"].split(".")
# check if keys exist in data Object and continue down levels till the value is reached
if key_path[0] in event_data["data"] and key["type"] == event_data["type"]:
print("inside first if")
value = event_data['data']
for sub_key in key_path:
print("inside loop")
if sub_key in value.keys():
print("in subkey" , sub_key)
value = value[sub_key]
# handle if keys and values dont exist
else:
value = None
break
if value is None:
continue
# determine where value is an Integer or a String and insert into correct column
val_int = None
val_str = None
if type(value) == str:
val_str = value
elif type(value) == int:
val_int = value
# add event_data row to session
insert = Event_data(event_id=event["id"], type=event_data["type"], key=key["key_name"], value_int=val_int, value_str=val_str )
print("insert object", insert)
db.session.add(insert)
def event_type_insert(event_type):
event_type_list = getattr(g, "event_type_list", None)
if event_type_list is None:
check = [item.event_type for item in Event_type.query.all()]
g.event_type_list = check
event_type_list = check
if event_type not in event_type_list:
insert = Event_type(event_type=event_type)
db.session.add(insert)
g.event_type_list.append(event_type)
I'm trying to pass a psycopg2 cursor to a tweepy stream.
The connection pool and cursors are configured in a seperate file. Only the cursor is passed as an argument to the main pipeline function in another file called get_tweet_topic.py. I need the cursor in the on_status() method because I have a query there that needs it for execution.
I cannot figure out how to pass the cursor onto the on_status() method in the MyStreamListener() class.
the error I get is:
2020-03-05T22:16:24.856945+00:00 app[worker.1]: self._target(*self._args, **self._kwargs)
2020-03-05T22:16:24.856945+00:00 app[worker.1]: File "/app/get_tweet_topic.py", line 81, in guess_topic_pipeline
2020-03-05T22:16:24.856946+00:00 app[worker.1]: status_streams.streaming_pipeline(api, cursor)
2020-03-05T22:16:24.856947+00:00 app[worker.1]: File "/app/status_streams.py", line 100, in streaming_pipeline
2020-03-05T22:16:24.856947+00:00 app[worker.1]: general_stream(api, cursor)
2020-03-05T22:16:24.856948+00:00 app[worker.1]: File "/app/status_streams.py", line 86, in general_stream
2020-03-05T22:16:24.856948+00:00 app[worker.1]: myStreamListener = MyStreamListener()
2020-03-05T22:16:24.856948+00:00 app[worker.1]: TypeError: __init__() missing 1 required positional argument: 'cursor'
Code:
status_streams.py:
import tweepy
import os
import db_queries
import follow
#define class for the stream listener
class MyStreamListener(tweepy.StreamListener):
def __init__(self, cursor):
super().__init__()
self.cursor = cursor
#set counter to only get 1200 tweets
self.counter = 0
self.max = 1200
#get tweets
def on_status(self, status):
if not status.retweeted:
status_dict = {'created_at': status.created_at.strftime('%y-%m-&d %H:%M'),
'source_stream': 'general stream',
'status_id': status.id_str,
'user_id': status.user.id_str,
'screen_name': status.user.name,
'tweet_text': status.text,
'num_likes': status.favorite_count,
'num_retweets': status.retweet_count}
created_at = status_dict['created_at']
source_stream = status_dict['source_stream']
status_id = status_dict['status_id']
user_id = status_dict['user_id']
screen_name = status_dict['screen_name']
tweet_text = status_dict['tweet_text']
num_likes = status_dict['num_likes']
num_retweets = status_dict['num_retweets']
db_queries.insert_raw_tweets_table(cursor, created_at, source_stream, status_id, user_id, screen_name, tweet_text, num_likes, num_retweets)
self.counter +=1
if self.counter == self.max:
return False
#get tweets from list of followers
def following_stream(api, cursor, user_name):
try:
for status in tweepy.Cursor(api.user_timeline, tweet_mode='extended', include_rts=False, screen_name=user_name).items(1):
#ignore retweets
if not status.retweeted:
status_dict = {'created_at': status.created_at.strftime('%y-%m-%d %H:%M'),
'source_stream': 'following stream',
'status_id': status.id_str,
'user_id': status.user.id_str,
'screen_name': status.user.name,
'tweet_text':status.full_text,
'num_likes':status.favorite_count,
'num_retweets':status.retweet_count}
created_at = status_dict['created_at']
source_stream = status_dict['source_stream']
status_id = status_dict['status_id']
user_id = status_dict['user_id']
screen_name = status_dict['screen_name']
tweet_text = status_dict['tweet_text']
num_likes = status_dict['num_likes']
num_retweets = status_dict['num_retweets']
db_queries.insert_raw_tweets_table(cursor, created_at, source_stream, status_id, user_id, screen_name, tweet_text, num_likes, num_retweets)
#function that controls both streams
def streaming_pipeline(api, cursor):
#get list of all users that are currently followed
#iterate through the following_list and grab the single latest tweet
following_list = follow.get_following(api)
for user in following_list:
f_stream = following_stream(api, cursor, user)
#stream class is used here
myStreamListener = MyStreamListener()
stream = tweepy.Stream(auth=api.auth, listener=myStreamListener(cursor=self.cursor))
stream.filter(languages=['en'], track=['the'])
cursor.close()
relevant section of get_tweet_topic.py:
def guess_topic_pipeline(api, conn, model, corpus, classifier):
while True:
cursor = conn.cursor()
db_queries.create_temp_tweets_table(cursor)
conn.commit()
#use pipeline to grab tweets off twitter
print('Retrieving statuses from streams...')
status_streams.streaming_pipeline(api, cursor)
print('Done retrieving...')
relevant portion of connection pooling code:
#get connection from pool, pass cursor as an argument, start topic extration thread
topic_conn = conn_pool.getconn()
topic_extraction_thread = Thread(target=get_tweet_topic.guess_topic_pipeline, kwargs={'api':api, 'conn': topic_conn, 'model': lda_model, 'corpus': lda_id2word, 'classifier': lda_huber_classifier})
topic_extraction_thread.start()
#return connection when done
conn_pool.putconn(topic_conn)
insert_raw_tweets_table() function with actual query:
def insert_raw_tweets_table(cursor, createdAt, sourceStream, statusID, userID, screenName, tweetText, numLikes, numRetweets):
cursor.execute('INSERT INTO tempTweets(createdAt, sourceStream, statusID, userID, screenName, tweetText) VALUES(%s, %s, %s, %s, %s, %s, %s, %s)', (createdAt, sourceStream, statusID, userID, screenName, tweetText, numLikes, numRetweets))
#MauriceMeyer answered the question in the comments but here is the working code for clarity.
I forgot to reference the cursor as self.cursor within the class and I forgot to pass the cursor as an argument when creating an instance of the class. I was passing the cursor as an argument after creating the instance, which is not correct.
Correct code:
class MyStreamListener(tweepy.StreamListener):
def __init__(self, cursor):
super().__init__()
self.cursor = cursor
#set counter to only get 1200 tweets
self.counter = 0
self.max = 1200
#get tweets
def on_status(self, status):
if not status.retweeted:
status_dict = {'created_at' : status.created_at.strftime('%y-%m-&d %H:%M'),
'source_stream' : 'general stream',
'status_id' : status.id_str,
'user_id' : status.user.id_str,
'screen_name' : status.user.name,
'tweet_text' : status.text,
'num_likes' : status.favorite_count,
'num_retweets' : status.retweet_count}
created_at = status_dict['created_at']
source_stream = status_dict['source_stream']
status_id = status_dict['status_id']
user_id = status_dict['user_id']
screen_name = status_dict['screen_name']
tweet_text = status_dict['tweet_text']
num_likes = status_dict['num_likes']
num_retweets = status_dict['num_retweets']
#▼ reference self.cursor here
db_queries.insert_raw_tweets_table(self.cursor, created_at, source_stream, status_id, user_id, screen_name, tweet_text, num_likes, num_retweets)
self.counter +=1
if self.counter == self.max:
return False
#stream class is used here ▼ reference cursor here
myStreamListener = MyStreamListener(cursor)
#▼ removed reference to cursor here
stream = tweepy.Stream(auth=api.auth, listener=myStreamListener)
stream.filter(languages=['en'], track=['the'])
I trying to add users in to database but stuck with this error:
[08/Oct/2018 14:18:01] - [O M N I A] Updated online for user: Майор пейн
[08/Oct/2018 14:18:01] - [O M N I A] Updated online for user: Nikodus
Exception in thread Thread-4:
Traceback (most recent call last):
File "/usr/lib/python3.5/threading.py", line 914, in _bootstrap_inner
self.run()
File "/usr/lib/python3.5/threading.py", line 862, in run
self._target(*self._args, **self._kwargs)
File "/root/bot/botevents.py", line 48, in counter
db_cursor.execute("INSERT INTO `users` (`server_id`, `user_name`, `user_id`, `last_update_online`, `last_update`) VALUES ('{0}', '{1}', '{2}', {3}, {3})".format(server.id, member.name.replace("'", ''), member.id, time.time()))
File "/usr/local/lib/python3.5/dist-packages/pymysql/cursors.py", line 170, in execute
result = self._query(query)
File "/usr/local/lib/python3.5/dist-packages/pymysql/cursors.py", line 328, in _query
conn.query(q)
File "/usr/local/lib/python3.5/dist-packages/pymysql/connections.py", line 516, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/usr/local/lib/python3.5/dist-packages/pymysql/connections.py", line 727, in _read_query_result
result.read()
File "/usr/local/lib/python3.5/dist-packages/pymysql/connections.py", line 1066, in read
first_packet = self.connection._read_packet()
File "/usr/local/lib/python3.5/dist-packages/pymysql/connections.py", line 683, in _read_packet
packet.check_error()
File "/usr/local/lib/python3.5/dist-packages/pymysql/protocol.py", line 220, in check_error
err.raise_mysql_exception(self._data)
File "/usr/local/lib/python3.5/dist-packages/pymysql/err.py", line 109, in raise_mysql_exception
raise errorclass(errno, errval)
pymysql.err.ProgrammingError: (1064, "You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '397911167944622100', 1539008281.6296663, 1539008281.6296663)' at line 1")
It's add some users but in one moment stuck and get that error.
It's possible bcz it's a 7k users and I have a VDS with only one core and 2 ram?
import pymysql, time, discord
from datetime import datetime
import warnings
warnings.simplefilter("ignore")
class BotEvents:
def __init__(self, *args, **kwargs):
#self.bot.event
async def on_ready():
self.log('\n', save=True)
self.log('-----------------------------------------', save=True)
self.log('Authorize successfully', save=True)
self.log('Authorized servers: ', save=True)
[self.log(' Name (id): %s (%s)' % (info.name, info.id), save=True) for info in self.bot.guilds]
self.log('-----------------------------------------', save=True)
await self.bot.loop.create_task(self.female_logic())
def counter(self, server):
db_conn, db_cursor = self.connect_db()
result = db_cursor.execute("SELECT `user_id` FROM `users` WHERE server_id = '{}'".format(server.id))
members = []
if result != 0:
for x in range(result):
members.append(db_cursor.fetchone()[0])
db_conn.close()
while self.run_c:
roles = []
member_role = []
db_conn, db_cursor = self.connect_db()
result = db_cursor.execute("SELECT `role_id`, `min_exp` FROM `auto_roles` WHERE server_id = '{}' ORDER BY `min_exp`".format(server.id))
if result != 0:
info = db_cursor.fetchone()
member_role = [str(info[0]), int(info[1])]
for x in range(result - 1):
info = db_cursor.fetchone()
roles.append([str(info[0]), int(info[1])])
db_conn.close()
for member in server.members:
if member != self.bot.user:
db_conn, db_cursor = self.connect_db()
if str(member.id) not in members:
db_cursor.execute("INSERT INTO `users` (`server_id`, `user_name`, `user_id`, `last_update_online`, `last_update`) VALUES ('{0}', '{1}', '{2}', {3}, {3})".format(server.id, member.name.replace("'", ''), member.id, time.time()))
db_conn.commit()
members.append(str(member.id))
self.log(' [%s] Added user: %s' % (server.name, member.name), save=True)
else:
db_cursor.execute("SELECT `member`, `balance`, `online_time`, `last_update_online`, `experience`, `last_update`, `autorole` FROM `users` WHERE (user_id = '{}' AND server_id = '{}')".format(member.id, server.id))
info = db_cursor.fetchone()
if info is not None:
info = {
'member':bool(info[0]),
'balance':int(info[1]),
'online_time':int(info[2]),
'last_update_online':int(info[3]),
'experience':int(info[4]),
'last_update':int(info[5]),
'autorole':info[6]
}
if int(time.time()) - info['last_update_online'] > 60:
if str(member.status) in ['online', 'idle', 'dnd']:
db_cursor.execute("UPDATE `users` SET online_time = {}, last_update_online = {} WHERE (user_id = '{}' AND server_id = '{}')".format(
info['online_time'] + 1,
time.time(),
member.id,
server.id))
db_conn.commit()
self.log('[%s] Updated online for user: %s ' % (server.name, member.name), save=True)
if int(time.time()) - info['last_update'] > 60:
if member.voice is not None:
db_cursor.execute("UPDATE `users` SET balance = {}, experience = {}, last_update = {} WHERE (user_id = '{}' AND server_id = '{}')".format(
info['balance'] + self.gold_per_minut,
info['experience'] + self.exp_per_minut,
time.time(),
member.id,
server.id))
db_conn.commit()
self.log('[%s] Updated balance, experience for user: %s ' % (server.name, member.name), save=True)
if len(member_role) != 0:
if not info['member'] and info['experience'] >= member_role[1]:
role_o = discord.utils.get(server.roles, id=int(member_role[0]))
if role_o not in member.roles:
self.bot.loop.create_task(member.add_roles(role_o))
db_cursor.execute("UPDATE `users` SET member = {} WHERE (user_id = '{}' AND server_id = '{}')".format(True, member.id, server.id))
db_conn.commit()
new_role = ''
for role in roles:
if role[1] <= info['experience']:
new_role = role[0]
if new_role != '' and new_role != info['autorole']:
if info['autorole'] != '0':
role_o = discord.utils.get(server.roles, id=int(info['autorole']))
if role_o in member.roles:
self.bot.loop.create_task(member.remove_roles(role_o))
self.log('[%s] For user %s deleted old role %s(%s)' % (
server.name,
member.name,
role_o.name,
role_o.id) , save=True)
time.sleep(2)
role_o = discord.utils.get(server.roles, id=int(new_role))
if role_o is not None:
if role_o not in member.roles:
db_cursor.execute("UPDATE `users` SET autorole = '{}' WHERE (user_id = '{}' AND server_id = '{}')".format(new_role, member.id, server.id))
db_conn.commit()
self.bot.loop.create_task(member.add_roles(role_o))
self.log('[%s] For user %s added role %s(%s)' % (server.name, member.name, role_o.name, role_o.id), save=True)
db_conn.close()
Someone knew how to fix this?
botevents.py file which get this error (pastebin snippet)
#Suresh, complete error after add quote:
[08/Oct/2018 16:37:33] - [O M N I A] Added user: Shein
[08/Oct/2018 16:37:33] - [O M N I A] Added user: ара
[08/Oct/2018 16:37:34] - [O M N I A] Added user: ROFLANOV
Exception in thread Thread-4:
Traceback (most recent call last):
File "/usr/lib/python3.5/threading.py", line 914, in _bootstrap_inner
self.run()
File "/usr/lib/python3.5/threading.py", line 862, in run
self._target(*self._args, **self._kwargs)
File "/root/bot/botevents.py", line 48, in counter
db_cursor.execute("INSERT INTO `users` (`server_id`, `user_name`, `user_id`, `last_update_online`, `last_update`) VALUES ('{0}', '{1}', '{2}', '{3}', '{3}')".format(server.id, member.name.replace("'", ''), member.id, time.time()))
File "/usr/local/lib/python3.5/dist-packages/pymysql/cursors.py", line 170, in execute
result = self._query(query)
File "/usr/local/lib/python3.5/dist-packages/pymysql/cursors.py", line 328, in _query
conn.query(q)
File "/usr/local/lib/python3.5/dist-packages/pymysql/connections.py", line 516, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/usr/local/lib/python3.5/dist-packages/pymysql/connections.py", line 727, in _read_query_result
result.read()
File "/usr/local/lib/python3.5/dist-packages/pymysql/connections.py", line 1066, in read
first_packet = self.connection._read_packet()
File "/usr/local/lib/python3.5/dist-packages/pymysql/connections.py", line 683, in _read_packet
packet.check_error()
File "/usr/local/lib/python3.5/dist-packages/pymysql/protocol.py", line 220, in check_error
err.raise_mysql_exception(self._data)
File "/usr/local/lib/python3.5/dist-packages/pymysql/err.py", line 109, in raise_mysql_exception
raise errorclass(errno, errval)
pymysql.err.ProgrammingError: (1064, "You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '397911167944622100', '1539016654.7659388', '1539016654.7659388')' at line 1")
You need to enclose time values in single quote while inserting data.
Please use code below:
import pymysql, time, discord
from datetime import datetime
import warnings
warnings.simplefilter("ignore")
class BotEvents:
def __init__(self, *args, **kwargs):
#self.bot.event
async def on_ready():
self.log('\n', save=True)
self.log('-----------------------------------------', save=True)
self.log('Authorize successfully', save=True)
self.log('Authorized servers: ', save=True)
[self.log(' Name (id): %s (%s)' % (info.name, info.id), save=True) for info in self.bot.guilds]
self.log('-----------------------------------------', save=True)
await self.bot.loop.create_task(self.female_logic())
def counter(self, server):
db_conn, db_cursor = self.connect_db()
result = db_cursor.execute("SELECT `user_id` FROM `users` WHERE server_id = '{}'".format(server.id))
members = []
if result != 0:
for x in range(result):
members.append(db_cursor.fetchone()[0])
db_conn.close()
while self.run_c:
roles = []
member_role = []
db_conn, db_cursor = self.connect_db()
result = db_cursor.execute("SELECT `role_id`, `min_exp` FROM `auto_roles` WHERE server_id = '{}' ORDER BY `min_exp`".format(server.id))
if result != 0:
info = db_cursor.fetchone()
member_role = [str(info[0]), int(info[1])]
for x in range(result - 1):
info = db_cursor.fetchone()
roles.append([str(info[0]), int(info[1])])
db_conn.close()
for member in server.members:
if member != self.bot.user:
db_conn, db_cursor = self.connect_db()
if str(member.id) not in members:
db_cursor.execute("INSERT INTO `users` (`server_id`, `user_name`, `user_id`, `last_update_online`, `last_update`) VALUES ('{0}', '{1}', '{2}', '{3}', '{3}')".format(server.id, member.name.replace("'", ''), member.id, time.time()))
db_conn.commit()
members.append(str(member.id))
self.log(' [%s] Added user: %s' % (server.name, member.name), save=True)
else:
db_cursor.execute("SELECT `member`, `balance`, `online_time`, `last_update_online`, `experience`, `last_update`, `autorole` FROM `users` WHERE (user_id = '{}' AND server_id = '{}')".format(member.id, server.id))
info = db_cursor.fetchone()
if info is not None:
info = {
'member':bool(info[0]),
'balance':int(info[1]),
'online_time':int(info[2]),
'last_update_online':int(info[3]),
'experience':int(info[4]),
'last_update':int(info[5]),
'autorole':info[6]
}
if int(time.time()) - info['last_update_online'] > 60:
if str(member.status) in ['online', 'idle', 'dnd']:
db_cursor.execute("UPDATE `users` SET online_time = {}, last_update_online = {} WHERE (user_id = '{}' AND server_id = '{}')".format(
info['online_time'] + 1,
time.time(),
member.id,
server.id))
db_conn.commit()
self.log('[%s] Updated online for user: %s ' % (server.name, member.name), save=True)
if int(time.time()) - info['last_update'] > 60:
if member.voice is not None:
db_cursor.execute("UPDATE `users` SET balance = {}, experience = {}, last_update = {} WHERE (user_id = '{}' AND server_id = '{}')".format(
info['balance'] + self.gold_per_minut,
info['experience'] + self.exp_per_minut,
time.time(),
member.id,
server.id))
db_conn.commit()
self.log('[%s] Updated balance, experience for user: %s ' % (server.name, member.name), save=True)
if len(member_role) != 0:
if not info['member'] and info['experience'] >= member_role[1]:
role_o = discord.utils.get(server.roles, id=int(member_role[0]))
if role_o not in member.roles:
self.bot.loop.create_task(member.add_roles(role_o))
db_cursor.execute("UPDATE `users` SET member = {} WHERE (user_id = '{}' AND server_id = '{}')".format(True, member.id, server.id))
db_conn.commit()
new_role = ''
for role in roles:
if role[1] <= info['experience']:
new_role = role[0]
if new_role != '' and new_role != info['autorole']:
if info['autorole'] != '0':
role_o = discord.utils.get(server.roles, id=int(info['autorole']))
if role_o in member.roles:
self.bot.loop.create_task(member.remove_roles(role_o))
self.log('[%s] For user %s deleted old role %s(%s)' % (
server.name,
member.name,
role_o.name,
role_o.id) , save=True)
time.sleep(2)
role_o = discord.utils.get(server.roles, id=int(new_role))
if role_o is not None:
if role_o not in member.roles:
db_cursor.execute("UPDATE `users` SET autorole = '{}' WHERE (user_id = '{}' AND server_id = '{}')".format(new_role, member.id, server.id))
db_conn.commit()
self.bot.loop.create_task(member.add_roles(role_o))
self.log('[%s] For user %s added role %s(%s)' % (server.name, member.name, role_o.name, role_o.id), save=True)
db_conn.close()
Hope this works!
How to import data into mysql from the given JSON file in Python? I'm getting error as
Traceback (most recent call last): File "C:/Users/risha/PycharmProjects/Test/JSON_TEST.py", line 24, in
con = pymysql.connect(host = '127.0.0.1',user = 'root',passwd = '',db = 'test',port ='3306') File
"C:\Users\risha\AppData\Local\Programs\Python\Python36\lib\site-packages\pymysql__init__.py",
line 90, in Connect
return Connection(*args, **kwargs) File "C:\Users\risha\AppData\Local\Programs\Python\Python36\lib\site-packages\pymysql\connections.py",
line 704, in init
self.connect() File "C:\Users\risha\AppData\Local\Programs\Python\Python36\lib\site-packages\pymysql\connections.py",
line 964, in connect
self.host_info = "socket %s:%d" % (self.host, self.port) TypeError: %d format: a number is required, not str
import pymysql
import os
import json
# read JSON file which is in the next parent folder
file = 'C://Users//risha//PycharmProjects//Test//video.json'
json_data= open(file).read()
json_obj = json.loads(json_data)
# do validation and checks before insert
def validate_string(val):
if val != None:
if type(val) is int:
#for x in val:
# print(x)
return str(val).encode('utf-8')
else:
return val
# connect to MySQL
con = pymysql.connect(host = '127.0.0.1',user = 'root',passwd = '',db = 'test',port ='3306')
cursor = con.cursor()
# parse json data to SQL insert
for i, item in enumerate(json_obj):
person = validate_string(item.get("person", None))
year = validate_string(item.get("year", None))
company = validate_string(item.get("company", None))
cursor.execute("INSERT INTO testp (person, year, company) VALUES (%s, %s, %s)", (person, year, company))
con.commit()
con.close()
Below code works perfectly in python2 with MySQLDB, how can I make it Python3 compatible?
I have debugged and searched for similar questions.
Error:
Exception ignored in: > Traceback (most recent call last): File > > > "/usr/local/lib/python3.4/dist-packages/pymysql/cursors.py", line 41, in del File > "/usr/local/lib/python3.4/dist-packages/pymysql/cursors.py", line 47, in close ReferenceError: weakly-referenced object no longer exists –
class Database():
def __init__(self):
self.host = 'localhost'
self.user = 'user'
self.password = 'pwd'
self.db = 'dbname'
self.connection = pymysql.connect(host=self.host, user=self.user, passwd=self.password, db=self.db,use_unicode=True, charset="utf8")
self.cursor = self.connection.cursor()
def storeToDB(self,ID,string,g,e):
import datetime
curr_time = datetime.datetime.now()
status = 0
try:
self.cursor.execute("""INSERT INTO master_data (`job_id`,`sstring`,`grl`,`erl`,`status`,`insert_timestamp`) \
VALUES (%s,%s,%s,%s,%s,%s)""",(jobID,search_string,g,e,status,curr_time))
self.connection.commit()
except:
self.connection.rollback()
if __name__ == "__main__":
db = Database()
db.storeToDB(20,"hello","something.com","e.com")
Try to do some opt before db operation finish :
cursor.close()
connection.close()