Related
I have an application where a service makes a POST request to a flask app which then inserts the body of request into a postgresql database into a table as well as inserting data into several other tables.
The problem occurs once a curtain amount of requests are made that the queue pool limit is reach and the app throws an error. i have isolated it down to the extra functions called to do the inserting into other tables. i don't understand what is happening.
Each service is a docker container
172.30.0.17 - - [16/Jun/2022 06:25:07] "GET /api/orgs HTTP/1.1" 500 -
Traceback (most recent call last):
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 2095, in __call__
return self.wsgi_app(environ, start_response)
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 2080, in wsgi_app
response = self.handle_exception(e)
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 2077, in wsgi_app
response = self.full_dispatch_request()
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 1525, in full_dispatch_request
rv = self.handle_user_exception(e)
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 1523, in full_dispatch_request
rv = self.dispatch_request()
File "/root/.local/lib/python3.9/site-packages/flask/app.py", line 1509, in dispatch_request
return self.ensure_sync(self.view_functions[rule.endpoint])(**req.view_args)
File "/service/routes/get/get_org.py", line 17, in base_route
query = [convert_class(item) for item in Organization.query.all()]
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/orm/query.py", line 2768, in all
return self._iter().all()
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/orm/query.py", line 2903, in _iter
result = self.session.execute(
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 1711, in execute
conn = self._connection_for_bind(bind)
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 1552, in _connection_for_bind
return self._transaction._connection_for_bind(
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 747, in _connection_for_bind
conn = bind.connect()
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3234, in connect
return self._connection_cls(self, close_with_result=close_with_result)
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 96, in __init__
else engine.raw_connection()
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3313, in raw_connection
return self._wrap_pool_connect(self.pool.connect, _connection)
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 3280, in _wrap_pool_connect
return fn()
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 310, in connect
return _ConnectionFairy._checkout(self)
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 868, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/pool/base.py", line 476, in checkout
rec = pool._do_get()
File "/root/.local/lib/python3.9/site-packages/sqlalchemy/pool/impl.py", line 134, in _do_get
raise exc.TimeoutError(
sqlalchemy.exc.TimeoutError: QueuePool limit of size 20 overflow 20 reached, connection timed out, timeout 30.00 (Background on this error at: https://sqlalche.me/e/14/3o7r)
Config
class Config:
# Database
SQLALCHEMY_DATABASE_URI = environ.get("DB_URI")
SQLALCHEMY_ECHO = False
SQLALCHEMY_TRACK_MODIFICATIONS = False
SQLALCHEMY_ENGINE_OPTIONS = {
"pool_size": 1,
"max_overflow": 0,
}
App init
db = SQLAlchemy()
# create flask App
def create_app():
app = Flask(__name__, instance_relative_config=False)
# DB config - postgreSQL
app.config['SQLALCHEMY_DATABASE_URI'] = getenv("DB_URI")
app.config.from_object('config.Config')
# config server with CORS
# cors = CORS(app, resources={r"/*": {"origins": "*"}})
with app.app_context():
db.init_app(app)
from service.routes.post import post_event
return app
app = create_app()
POST request
#app.route('/api/orgs/<org_id>/sites/<site_id>/sessions/<session_id>/events', methods=["POST"])
def edger_post_request(org_id, site_id, session_id):
print("URL params: {}, {}, {}".format(
org_id, site_id, session_id
))
# parse body of post request
try:
request_body = request.json
event_data = json.loads(request_body["data"])
except:
return "error in body of request", 400
print("Body of request: ", request_body)
# check if body of request is valid
errors_body, has_error = validate_input(request_body)
print(errors_body, has_error)
if has_error:
return Response(errors_body, 400)
# insert event
event_dict = dict()
try:
event_dict = {
"id": request_body["id"],
"timestamp": request_body["timestamp"],
"type": request_body["type"],
"event_type": event_data["type"],
"data": request_body["data"],
"org_id": org_id,
"site_id": site_id,
"device_id": request_body["device_id"],
"session_id": request_body["session_id"]
}
if "frame_url" in request_body.keys():
event_dict["frame_url"] = request_body["frame_url"]
else:
event_dict["frame_url"] = None
insert_event = Event(**event_dict)
db.session.add(insert_event)
# commit event to DB
db.session.commit()
# Insert extra tables
insert_threads(org_id, site_id, event_dict, event_data)
db.session.commit()
db.session.close()
return {"result": "success"}, 200
except Exception as e:
print("- - - - - - - - - - - - - - - - -")
db.session.rollback()
print("Error inserting Event", e)
print("- - - - - - - - - - - - - - - - -")
db.session.close()
return {"result": "unsuccessful"}, 409
insert_threads function
def insert_threads(org_id, site_id, body, data):
event_data_insert(body, data),
event_type_insert(data["type"]),
insert_org(org_id),
insert_site(org_id, site_id),
insert_device(org_id, site_id, body['device_id']),
insert_session(org_id, site_id, body['device_id'], body['session_id'], body['timestamp'])
def insert_org(org_id):
org_list = getattr(g, "org_list", None)
# query organization insert if doesn't exist
if org_list is None:
check = [org.id for org in Organization.query.all()]
g.org_list = check
org_list = check
if org_id not in org_list:
insert_org = Organization(id=org_id, name="temp_name")
db.session.add(insert_org)
org_list.append(insert_org.id)
g.org_list = org_list
def insert_site(org_id, site_id):
site_list = getattr(g, "site_list", None)
# query Site insert if doesn't exist
if site_list is None:
check = [site.id for site in Site.query.all()]
g.site_list = check
site_list = check
if site_id not in site_list:
insert_site = Site(id=site_id, org_id=org_id, name="temp_name")
db.session.add(insert_site)
site_list.append(insert_site.id)
g.site_list = site_list
def insert_device(org_id, site_id, device_id):
device_list = getattr(g, "device_list", None)
# query device insert if doesn't exist
if device_list is None:
check = [device.id for device in Device.query.all()]
g.device_list = check
device_list = check
if device_id not in device_list:
insert_device = Device(id=device_id, site_id=site_id, org_id=org_id)
db.session.add(insert_device)
device_list.append(insert_device.id)
g.device_list = device_list
def insert_session(org_id, site_id, device_id, session_id, timestamp):
session_list = getattr(g, "session_list", None)
# query session and insert if it doesn't already exist
if session_list is None:
check = [session.id for session in Session.query.all()]
g.session_list = check
session_list = check
if session_id not in session_list:
insert_session = Session(
id=session_id,
timestamp=timestamp,
org_id=org_id,
site_id=site_id,
device_id=device_id
)
db.session.add(insert_session)
session_list.append(insert_session.id)
g.session_list = session_list
def event_data_insert(event, event_data):
# get white list from global var else None
query_list = getattr(g, "query_list", None)
# if query_list is None get query_list from DB
if query_list is None:
check = [convert_class(item) for item in Event_data_query_list.query.all()]
g.query_list = check
query_list = check
# loop though query_list and insert key/value pair into event_data table
for key in query_list:
print("the key: ", key)
key_path = key["key_name"].split(".")
# check if keys exist in data Object and continue down levels till the value is reached
if key_path[0] in event_data["data"] and key["type"] == event_data["type"]:
print("inside first if")
value = event_data['data']
for sub_key in key_path:
print("inside loop")
if sub_key in value.keys():
print("in subkey" , sub_key)
value = value[sub_key]
# handle if keys and values dont exist
else:
value = None
break
if value is None:
continue
# determine where value is an Integer or a String and insert into correct column
val_int = None
val_str = None
if type(value) == str:
val_str = value
elif type(value) == int:
val_int = value
# add event_data row to session
insert = Event_data(event_id=event["id"], type=event_data["type"], key=key["key_name"], value_int=val_int, value_str=val_str )
print("insert object", insert)
db.session.add(insert)
def event_type_insert(event_type):
event_type_list = getattr(g, "event_type_list", None)
if event_type_list is None:
check = [item.event_type for item in Event_type.query.all()]
g.event_type_list = check
event_type_list = check
if event_type not in event_type_list:
insert = Event_type(event_type=event_type)
db.session.add(insert)
g.event_type_list.append(event_type)
I'm making telegram bot, which is adding some messages to mysql database and then deleting them. When I use this bot only by myself I don't get any errors, but if I sending messages using second account and trying to do it at the same time, after 5-10 messages I get "weakly-referenced object no longer exists" error. I guess I somehow lose connection to mysql database but I don't understand why. That's the error.
2020-08-10 00:05:48,249 (util.py:68 WorkerThread1) ERROR - TeleBot: "ReferenceError occurred, args=('weakly-referenced object no longer exists',)
Traceback (most recent call last):
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\telebot\util.py", line 62, in run
task(*args, **kwargs)
File "C:/shit stuff/code stuff/projects/dota_tg_bot/tg_bot.py", line 47, in process_quiz_second_question
db.add_answer(user_id, int(message.text), 2)
File "C:\shit stuff\code stuff\projects\dota_tg_bot\database.py", line 129, in add_answer
self.cursor.execute(add_answer_statement, (answer, date, user_id))
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\mysql\connector\cursor.py", line 569, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\mysql\connector\cursor.py", line 484, in _handle_result
self._connection.unread_result = False
ReferenceError: weakly-referenced object no longer exists
"
Traceback (most recent call last):
File "C:/shit stuff/code stuff/projects/dota_tg_bot/tg_bot.py", line 136, in <module>
main()
File "C:/shit stuff/code stuff/projects/dota_tg_bot/tg_bot.py", line 132, in main
bot.polling()
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\telebot\__init__.py", line 427, in polling
self.__threaded_polling(none_stop, interval, timeout)
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\telebot\__init__.py", line 451, in __threaded_polling
self.worker_pool.raise_exceptions()
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\telebot\util.py", line 111, in raise_exceptions
six.reraise(self.exc_info[0], self.exc_info[1], self.exc_info[2])
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\six.py", line 703, in reraise
raise value
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\telebot\util.py", line 62, in run
task(*args, **kwargs)
File "C:/shit stuff/code stuff/projects/dota_tg_bot/tg_bot.py", line 47, in process_quiz_second_question
db.add_answer(user_id, int(message.text), 2)
File "C:\shit stuff\code stuff\projects\dota_tg_bot\database.py", line 129, in add_answer
self.cursor.execute(add_answer_statement, (answer, date, user_id))
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\mysql\connector\cursor.py", line 569, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "C:\shit stuff\code stuff\projects\dota_tg_bot\venv\lib\site-packages\mysql\connector\cursor.py", line 484, in _handle_result
self._connection.unread_result = False
ReferenceError: weakly-referenced object no longer exists
Process finished with exit code 1
That's class of mysql handler i use.
import mysql.connector
from config import DB_PASSWORD
from dota_parser import get_winrates, get_quiz_answers
import datetime
class MySQLDatabase:
def __init__(self):
self.connection = None
self.cursor = None
def connect(self):
self.connection = mysql.connector.connect(
host='localhost',
user='root',
password=DB_PASSWORD,
port='3306',
database='dotatelegrambot'
)
self.cursor = self.connection.cursor()
def close(self):
self.cursor.close()
self.connection.close()
def get_hero_answers(self, hero):
self.connect()
get_hero_answers_statement = ('SELECT Answer1, Answer2, Answer3 FROM heroes '
'WHERE Hero = %s')
self.cursor.execute(get_hero_answers_statement, (hero,))
answers = self.cursor.fetchone()
self.close()
return answers
def get_hero_winrates(self, hero):
self.connect()
get_winrate_statement = ('SELECT Winrate1, Winrate2, Winrate3, Winrate4, Winrate5 '
'FROM heroes '
'WHERE Hero = %s')
self.cursor.execute(get_winrate_statement, (hero,))
winrates = self.cursor.fetchone()
self.close()
return winrates
def get_heroes(self):
self.connect()
get_heroes_statement = 'SELECT Hero FROM heroes'
self.cursor.execute(get_heroes_statement)
heroes = self.cursor.fetchall()
self.close()
return heroes
def update_user(self, user_id, hero):
self.connect()
get_user_statement = ('SELECT UserID '
'FROM users '
'WHERE UserID = %s')
self.cursor.execute(get_user_statement, (user_id,))
user = self.cursor.fetchone()
now = datetime.datetime.now()
date = f'{now.year}-{now.month}-{now.day}'
if user:
update_user_statement = ('UPDATE users '
'SET CurrentHero = %s, LastDate = %s '
'WHERE UserID = %s')
self.cursor.execute(update_user_statement, (hero, date, user_id))
self.connection.commit()
else:
add_user_statement = ('INSERT INTO users (UserID, CurrentHero, LastDate) '
'VALUES (%s, %s, %s)')
self.cursor.execute(add_user_statement, (user_id, hero, date))
self.connection.commit()
self.close()
def get_user_hero(self, user_id):
self.connect()
get_user_hero_statement = ('SELECT CurrentHero '
'FROM users '
'WHERE UserID = %s')
self.cursor.execute(get_user_hero_statement, (user_id,))
hero = self.cursor.fetchone()[0]
self.close()
return hero
def add_answer(self, user_id, answer, number):
self.connect()
now = datetime.datetime.now()
date = f'{now.year}-{now.month}-{now.day}'
add_answer_statement = ('UPDATE users '
f'SET Answer{number} = %s, LastDate = %s '
'WHERE UserID = %s')
self.cursor.execute(add_answer_statement, (answer, date, user_id))
self.connection.commit()
self.close()
def get_user_answers(self, user_id):
self.connect()
get_user_answers_statement = ('SELECT Answer1, Answer2, Answer3 FROM users '
'WHERE UserID = %s')
self.cursor.execute(get_user_answers_statement, (user_id,))
answers = self.cursor.fetchone()
self.close()
return answers
def clear_answers(self, user_id):
self.connect()
now = datetime.datetime.now()
date = f'{now.year}-{now.month}-{now.day}'
clear_answers_statement = ('UPDATE users '
'SET CurrentHero = Null, Answer1 = Null, Answer2 = Null, Answer3 = Null, '
'LastDate = %s '
'WHERE UserID = %s')
self.cursor.execute(clear_answers_statement, (date, user_id))
self.connection.commit()
self.close()
def main():
db = MySQLDatabase()
db.connect()
db.close()
if __name__ == '__main__':
main()
Bot code.
import telebot
from telebot import types
import random
from config import TG_TOKEN, DOTA_RANKS
from database import MySQLDatabase
bot = telebot.TeleBot(TG_TOKEN)
db = MySQLDatabase()
#bot.message_handler(commands=['start'])
def welcome(message):
bot.send_message(message.from_user.id, 'Hi, I\'m dotabuff bot.')
bot.send_message(message.from_user.id, 'What do you want to do?', reply_markup=welcome_keyboard())
#bot.message_handler(commands=['quiz'])
def start_quiz(message):
msg = bot.send_message(message.from_user.id, 'Choose hero:', reply_markup=heroes_keyboard())
bot.register_next_step_handler(msg, process_quiz_choose_hero)
def process_quiz_choose_hero(message):
if cancel_check(message):
hero = message.text
user_id = int(message.from_user.id)
db.update_user(user_id, hero)
msg = bot.send_message(message.from_user.id, f'What is starting move speed of {hero}?',
reply_markup=quiz_answers_keyboard(db.get_hero_answers(hero)[0]))
bot.register_next_step_handler(msg, process_quiz_first_question)
def process_quiz_first_question(message):
if cancel_check(message):
user_id = int(message.from_user.id)
hero = db.get_user_hero(user_id)
db.add_answer(user_id, int(message.text), 1)
msg = bot.send_message(message.from_user.id, f'What is starting attack speed of {hero}?',
reply_markup=quiz_answers_keyboard(db.get_hero_answers(hero)[1]))
bot.register_next_step_handler(msg, process_quiz_second_question)
def process_quiz_second_question(message):
if cancel_check(message):
user_id = int(message.from_user.id)
hero = db.get_user_hero(user_id)
db.add_answer(user_id, int(message.text), 2)
msg = bot.send_message(message.from_user.id, f'What is starting attack range of {hero}?',
reply_markup=quiz_answers_keyboard(db.get_hero_answers(hero)[2]))
bot.register_next_step_handler(msg, process_quiz_third_question)
def process_quiz_third_question(message):
if cancel_check(message):
user_id = int(message.from_user.id)
hero = db.get_user_hero(user_id)
db.add_answer(user_id, int(message.text), 3)
correct_ans = 0
for ans_num in range(3):
correct_ans += db.get_hero_answers(hero)[ans_num] == db.get_user_answers(user_id)[ans_num]
if correct_ans != 1:
bot.send_message(user_id, f'Congratulations!!! You have exactly {correct_ans} correct answers.',
reply_markup=welcome_keyboard())
else:
bot.send_message(user_id, f'Congratulations!!! You have exactly {correct_ans} correct answer.',
reply_markup=welcome_keyboard())
db.clear_answers(user_id)
#bot.message_handler(commands=['winrates'])
def start_wr_check(message):
msg = bot.send_message(message.from_user.id, 'Choose hero:', reply_markup=heroes_keyboard())
bot.register_next_step_handler(msg, process_hero_winrate)
def process_hero_winrate(message):
if cancel_check(message):
hero = message.text
reply = ''
heroes_winrates = db.get_hero_winrates(hero)
for i, rank in enumerate(DOTA_RANKS):
line = f'{rank} — *{str(heroes_winrates[i])} %* \n'
reply += line
bot.send_message(message.from_user.id, reply, parse_mode='Markdown', reply_markup=welcome_keyboard())
#bot.message_handler(func=lambda message: message.text == 'Cancel ⛔️')
def menu(message):
bot.send_message(message.from_user.id, 'What do you want to do?', reply_markup=welcome_keyboard())
def welcome_keyboard():
markup = types.ReplyKeyboardMarkup(one_time_keyboard=True, resize_keyboard=True)
markup.add('/winrates', '/quiz')
return markup
def heroes_keyboard():
markup = types.ReplyKeyboardMarkup(one_time_keyboard=True, resize_keyboard=True)
markup.add('Cancel ⛔️')
heroes = db.get_heroes()
for hero in heroes:
markup.add(hero[0])
return markup
def quiz_answers_keyboard(correct_answer):
markup = types.ReplyKeyboardMarkup(one_time_keyboard=True, resize_keyboard=True)
markup.add('Cancel ⛔️')
answer_buttons = [types.KeyboardButton(correct_answer)]
int_ans = int(correct_answer)
for _ in range(2):
rand_answers = [x for x in range(int_ans - 45, int_ans - 4, 5)] +\
[x for x in range(int_ans + 5, int_ans + 46, 5)]
rand_ans = rand_answers.pop(random.randrange(len(rand_answers)))
answer_buttons.append(types.KeyboardButton(rand_ans))
random.shuffle(answer_buttons)
for button in answer_buttons:
markup.add(button)
return markup
def cancel_check(message):
if message.text == 'Cancel ⛔️':
bot.clear_step_handler_by_chat_id(message.from_user.id)
bot.send_message(message.from_user.id, 'What do you want to do?', reply_markup=welcome_keyboard())
return False
return True
def main():
bot.polling()
if __name__ == '__main__':
main()
So every time I need to take or input data in database I open connection and after it I close it. (There is a lot of non problem related stuff use Ctrl+F "db" to follow database methods calls.)
https://imgur.com/a/hSrAE4D
That's the structure of db for better understanding.
I'm not familiar enough with database related python's stuff, but I've seen in your code that every method you defined in MySqlDatabase creates a new instance of the database connection.
Are you sure you don't need to check if there is no connection open - to avoid instantiating a new one?
Here is how I would do:
def connect(self):
self.connection = mysql.connector.connect(
host='localhost',
user='root',
password=DB_PASSWORD,
port='3306',
database='dotatelegrambot'
)
def getCursor(self)
if not self.connection:
self.connect()
return self.connection.cursor()
def get_user_hero(self, user_id):
cursor = self.getCursor()
get_user_hero_statement = ('SELECT CurrentHero '
'FROM users '
'WHERE UserID = %s')
cursor.execute(get_user_hero_statement, (user_id,))
hero = cursor.fetchone()[0]
cursor.close()
return hero
Im trying convert to str
import mysql.connector
from mysql.connector import Error
def VeriEkleme(Deger1,Deger2):
try:
connection = mysql.connector.connect(host='localhost',database='pythonregister',user='pyroot',password='')
if connection.is_connected():
print("MySQL bağlantısı aktif edildi.")
mySql_insert_query = """INSERT INTO userinformations (Username, Password) VALUES """(Deger1,Deger2)
cursor = connection.cursor()
result = cursor.execute(mySql_insert_query)
connection.commit()
print("Record inserted successfully into Laptop table")
cursor.close()
except Error as e:
print("Error while connecting to MysqL", e)
def Register():
Username = input("Username:")
Password = input("Pass:")
VeriEkleme(str(Username),str(Password))
def EnterSystem():
Login = "login"
Answer = input("Login or Register?: ").lower()
if Answer == Login:
print("eşit")
Register()
else:
EnterSystem()
EnterSystem()
Login or Register?: login
eşit
Username:s
Pass:s
MySQL bağlantısı aktif edildi.
Traceback (most recent call last):
File "C:/Users/Relov/PycharmProjects/PygameProje1/PygameProjesi.py", line 36, in <module>
EnterSystem()
File "C:/Users/Relov/PycharmProjects/PygameProje1/PygameProjesi.py", line 32, in EnterSystem
Register()
File "C:/Users/Relov/PycharmProjects/PygameProje1/PygameProjesi.py", line 24, in Register
VeriEkleme(str(Username),str(Password))
File "C:/Users/Relov/PycharmProjects/PygameProje1/PygameProjesi.py", line 11, in VeriEkleme
mySql_insert_query = """INSERT INTO userinformations (Username, Password) VALUES """(Deger1,Deger2)
TypeError: 'str' object is not callable
Process finished with exit code 1
You're calling str as it is a function.
"""INSERT INTO userinformations (Username, Password) VALUES """(Deger1,Deger2)
I recommend using Prepared Statements. This is safe against SQL Injection attacks.
mySql_insert_query = """INSERT INTO userinformations (Username, Password) VALUES (%s, %s)"""
cursor = connection.cursor()
result = cursor.execute(mySql_insert_query, (Deger1, Deger2))
I need to Copy every mail from my 'inbox'+sub folders to 2 new folders, one for short subjects and another for the long ones.
The crawl always abort after the 10th mail, and I need to repeat the process for 10 000 of them.. if someone had a solution I would be very grateful, here is my code :
import imaplib, getpass, re, email
pattern_uid = re.compile(b'\d+ \(UID (?P<uid>\d+)\)')
def parse_mailbox(data):
flags, b, c = data.partition(' ')
separator, b, name = c.partition(' ')
return (flags, separator.replace('"', ''), name.replace('"', ''))
def connect(email):
imap = imaplib.IMAP4_SSL("outlook.office365.com",993)
password = "*******"
imap.login(email, password)
return imap
def disconnect(imap):
imap.logout()
def parse_uid(data):
match = pattern_uid.match(data)
return match.group('uid')
if __name__ == '__main__':
imap = connect('************')
rv,data = imap.list()
# print(data)
if rv == 'OK':
for mbox in data:
flags,separator,name = parse_mailbox(bytes.decode(mbox))
print(name)
if 'HasNoChildren' in flags and '2' in name:
name = name.replace('/ Inbox','Inbox')
rv2, data2 = imap.select('"'+name+'"')
print(rv2)
resp, items = imap.search(None, 'All')
email_ids = items[0].split()
# print(items)
print(email_ids)
# Assuming that you are moving the latest email.
# for mail in email_ids:
# if
print(len(email_ids))
for i in range(0,len(email_ids)):
print('NOUVEAU MESSAGE \n')
print(email_ids[i])
resp, data = imap.fetch(email_ids[i], "(UID)")
rv,sujet = imap.fetch(email_ids[i],("(RFC822)"))
varSubject= ""
for response_part in sujet:
if isinstance(response_part, tuple):
msg = email.message_from_bytes(response_part[1])
varSubject = msg['subject']
break
print(varSubject)
print(i)
print(type(data[0]))
print(data[0])
try :
msg_uid = parse_uid(data[0])
except AttributeError:
continue
num = ""
dossier = ""
if len(varSubject) <= 20:
dossier = 'INBOX/sujets_courts'
imap.create(dossier)
elif len(varSubject) > 20:
dossier = 'INBOX/sujets_longs'
imap.create(dossier)
print(varSubject+'******'+dossier)
print(msg_uid)
result = imap.uid('COPY', msg_uid, dossier)
# if result[0] == 'OK':
# mov, data = imap.uid('STORE', msg_uid , '+FLAGS', '(\Deleted)')
# imap.expunge()
disconnect(imap)
The email and the password are substituted by '****'
Thanks a lot :)
console output :
Traceback (most recent call last):
File "C:\Users\lschaub\AppData\Local\Programs\Python\Python36-32\lib\imaplib.py", line 1011, in _command_complete
typ, data = self._get_tagged_response(tag)
File "C:\Users\lschaub\AppData\Local\Programs\Python\Python36-32\lib\imaplib.py", line 1131, in _get_tagged_response
self._get_response()
File "C:\Users\lschaub\AppData\Local\Programs\Python\Python36-32\lib\imaplib.py", line 1039, in _get_response
resp = self._get_line()
File "C:\Users\lschaub\AppData\Local\Programs\Python\Python36-32\lib\imaplib.py", line 1143, in _get_line
raise self.abort('socket error: EOF')
imaplib.abort: socket error: EOF
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "movemail.py", line 44, in <module>
resp, data = imap.fetch(email_ids[i], "(UID)")
File "C:\Users\lschaub\AppData\Local\Programs\Python\Python36-32\lib\imaplib.py", line 526, in fetch
typ, dat = self._simple_command(name, message_set, message_parts)
File "C:\Users\lschaub\AppData\Local\Programs\Python\Python36-32\lib\imaplib.py", line 1188, in _simple_command
return self._command_complete(name, self._command(name, *args))
File "C:\Users\lschaub\AppData\Local\Programs\Python\Python36-32\lib\imaplib.py", line 1013, in _command_complete
raise self.abort('command: %s => %s' % (name, val))
imaplib.abort: command: FETCH => socket error: EOF
OK I figured out that the problem comes from the creation of an already-created folder, but I could not find any folder.exists() look-a-like method in the imaplib doc...
I'm writing a script, where I have to dump some columns of tables from an SQL database into a file, and transfer it via FTP.
Because dumps can get really big, my Idea was to write a FakeFile which querys row by row from a cursor in its readline method and pass it to ftplib.FTP.storlines.
This is what I have so far:
import ftplib
import MySQLdb
def MySQLFakeFile(object):
'''
Simulates a read-only file, which dumps rows on demand.
Use this, to pass it to the FTP protocol to make the dump more efficient,
without the need to dump it somewhere and copy it over the net afterwords
'''
def __init__(self, cursor, delimeter, table_name, query):
self.cursor = cursor
self.delimeter = delimeter
self.table_name = table_name
#query is something along select ... FROM %s
self.cursor.execute(query, table_name)
self._has_written_index = False
#file attrs
self.closed = False
self.name = table_name + ".csv"
self.encoding = "utf-8"
self.mode = "r"
def close(self):
self.cursor.close()
self.closed = True
def flush(self):
'''No-OP'''
pass
def read(self, size):
pass
def readline(self, size):
if not self._has_written_index:
ret = []
for desc in self.cursor.description:
ret.append(desc[0])
self._has_written_index = True
else:
ret = self.cursor.fetchone()
if not ret:
return None
s = ""
for col in ret:
s += str(col) + self.delimeter
return s + "\n"
def readlines(self, size):
ret = []
line = self.readline()
while line:
ret.append(line)
line = self.readline()
def write(self, string):
raise Exception("cannot write to a MySQLFakeFile")
def writelines(self, lines)
raise Exception("cannot write to a MySQLFakeFile")
db = MySQLdb("host", "user", "pass", "db")
ftp = ftplib.FTP("host", "user", "pass")
fakeFile = MySQLFakeFile(db.cursor(), ";", "tableName", "SELECT * FROM %s")
ftp.storlines("STOR tableName.csv", fakeFile)
gives me
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/lib/python2.7/ftplib.py", line 496, in storlines
if len(buf) > self.maxline:
TypeError: object of type 'NoneType' has no len()
What am I doing wrong, and whats the NoneType here?
Your readline returns None instead of empty string "" when reached end-of-rows.
Your readlines returns nothing.
def readlines(self, size):
ret = []
while True:
line = self.readline()
if not line:
break
ret.append(line)
return ret