I am attempting to use the following Lambda Function on AWS to automatically import objects from an S3 bucket into an RDS instance as they are uploaded into the bucket.
import boto3
import pymysql
s3_cient = boto3.client('s3')
# Read CSV file content from S3 bucket
def read_data_from_s3(event):
bucket_name = event["Records"][0]["s3"]["bucket"]["name"]
s3_file_name = event["Records"][0]["s3"]["object"]["key"]
resp = s3_cient.get_object(Bucket="bucket_name", Key=s3_file_name)
data = resp['Body'].read().decode('utf-8')
data = data.split("\n")
return data
def lambda_handler(event, context):
rds_endpoint = "rds-endpoint"
username = "name"
password = "pass" # RDS Mysql password
db_name = "db-name" # RDS MySQL DB name
conn = None
try:
conn = pymysql.connect(rds_endpoint, user=username, passwd=password, db=db_name, connect_timeout=5)
except pymysql.MySQLError as e:
print("ERROR: Unexpected error: Could not connect to MySQL instance.")
try:
cur = conn.cursor()
cur.execute("INSERT INTO `db`.`table`(`sensor`,`sIP`,`dIP`,`sPort`) VALUES(<{sensor: }>,<{sIP: }>,<{dIP: }>,<{sPort: }>);")
conn.commit()
except:
pass
data = read_data_from_s3(event)
with conn.cursor() as cur:
for ent in data: # Iterate over S3 csv file content and insert into MySQL database
try:
ent = ent.replace("\n","").split(",")
print (">>>>>>>"+str(ent))
cur.execute('insert into db`.`table (sensor,Sip,Dip,sPort) values("'+str(ent[1])+'")')
conn.commit()
except:
continue
cur.execute("select * from table")
# Display table records
for row in cur:
print (row)
if conn:
conn.commit()
Whenever I run this, I get the following error and I do not understand what I am doing wrong? Would someone be able to clarify my errors?
[ERROR] TypeError: __init__() takes 1 positional argument but 2 positional arguments (and 4 keyword-only arguments) were given
Traceback (most recent call last):
File "/var/task/lambda_function.py", line 23, in lambda_handler
conn = pymysql.connect(rds_endpoint, user=username, passwd=password, db=db_name, connect_timeout=5)
It seems that pymysql.connect only takes in keyword arguments. You are passing rds_endpoint in as a positional argument. The rds_endpoint argument should probably passed in as host=rds_endpoint.
Related
I'm writing a python code to read from mysql database:
def create_server_connection(host, user, password):
connection = None
try:
connection = pymysql.connect(host='localhost',
user='root',
password='pwd',
database='raw_data',
cursorclass=pymysql.cursors.DictCursor)
print("MySQL Database connection successful")
except err as error:
print(f"Error: '{error}'")
return connection
def read_query(connection, query):
cur = connection.cursor()
result = None
try:
cur.execute(query)
result = cur.fetchall()
return result
except err as error:
print(f"Error: '{error}'")
return cur
def get_Tables_byName(cursor, tableName):
q1 = f'''
SELECT table_name FROM raw_data.tables
where table_name like '{tableName}'; '''
res = []
cursor.execute(q1)
for row in cursor:
res.append(row[0])
return res
get_Tables_byName(cursor,'data_31942010201')
If I want to call get_Tables_byName function, what should I put in the first parameter? If I put cursor, the error message shows NameError: name 'cursor' is not defined
I have 2 python functions that handle an event in a lambda function that are essentially the same thing. When checking the logs in AWS I get the following error:
{
"errorMessage": "local variable 'post_connection' referenced before assignment",
"errorType": "UnboundLocalError",
"stackTrace": [
" File \"/var/task/etl_python/handler.py\", line 11, in handle\n EtlCall.run_bc(event)\n",
" File \"/var/task/etl_python/elt_call.py\", line 153, in run_bc\n if post_connection:\n"
]
}
My code looks like this:
def run_bo(event):
s3_resource = boto3.resource('s3')
idv_endpoint = os.getenv('DB_ENDPOINT')
idv_database = os.getenv("DB_NAME")
filename = 'staging/aml_bo'
bucket = os.getenv('BILLING_ETL')
if 'resources' in event and "psql_billing" in event['resources'][0]:
try:
config = VaultService()
s3_resource = boto3.resource('s3')
idv_endpoint = os.getenv('DB_ENDPOINT')
idv_database = os.getenv("DB_NAME")
filename = 'staging/billing_bo'
bucket = os.getenv('BILLING_ETL')
idv_username = config.get_postgres_username()
idv_password = config.get_postgres_password()
post_connection = psycopg2.connect(user = idv_username
, password = idv_password
, host = idv_endpoint
, port = "5432"
, database = idv_database)
cursor = post_connection.cursor()
bo_qry = "SELECT uuid\
,first_name, middle_initial, last_name, date(date_of_birth)
mailing_address, public.bo"
#Might need not need the next two lines but this should work.
query = """COPY ({}) TO STDIN WITH (FORMAT csv, DELIMITER '|', QUOTE '"', HEADER TRUE)""".format(bo_qry)
file = StringIO()
cursor.copy_expert(query, file)
s3_resource.Object(bucket, f'{filename}.csv').put(Body=file.getvalue())
cursor.close()
except(Exception, psycopg2.Error) as error:
print("Error connecting to postgres instance", error)
finally:
if post_connection:
cursor.close()
post_connection.close()
#return "SUCCESS"
else:
# Unknown notification
#raise Exception(f'Unexpected event notification: {event}')
print("Cannot make a solid connection to psql instance. Please check code configuration")
def run_bc(event):
if 'resources' in event and "psql_billing" in event['resources'][0]:
try:
config = VaultService()
s3_resource = boto3.resource('s3')
idv_endpoint = os.getenv('DB_ENDPOINT')
idv_database = os.getenv("DB_NAME")
filename = 'staging/billing_bc'
bucket = os.getenv('BILLING_ETL')
idv_username = config.get_postgres_username()
idv_password = config.get_postgres_password()
post_connection = psycopg2.connect(user = idv_username
, password = idv_password
, host = idv_endpoint
, port = "5432"
, database = idv_database)
cursor = post_connection.cursor()
bc_qry = "select id, uuid, document_type, image_id,
document_id\
from public.bc"
#Might need not need the next two lines but this should work.
query = """COPY ({}) TO STDIN WITH (FORMAT csv, DELIMITER '|', QUOTE '"', HEADER TRUE)""".format(bc_flowdown_qry)
file = StringIO()
cursor.copy_expert(query, file)
s3_resource.Object(bucket, f'{filename}.csv').put(Body=file.getvalue())
cursor.close()
except(Exception, psycopg2.Error) as error:
print("Error connecting to postgres instance", error)
finally:
if post_connection:
cursor.close()
post_connection.close()
#return "SUCCESS"
else:
# Unknown notification
#raise Exception(f'Unexpected event notification: {event}')
print("Cannot make a solid connection to psql instance. Please check code configuration")
I don't understand how my connection is unbound if I am closing the connection and the connection after each function and then reopening it for the next. I close it at the end when the data is dumped to my file and then create a new connection in the next function.
Im trying convert to str
import mysql.connector
from mysql.connector import Error
def VeriEkleme(Deger1,Deger2):
try:
connection = mysql.connector.connect(host='localhost',database='pythonregister',user='pyroot',password='')
if connection.is_connected():
print("MySQL bağlantısı aktif edildi.")
mySql_insert_query = """INSERT INTO userinformations (Username, Password) VALUES """(Deger1,Deger2)
cursor = connection.cursor()
result = cursor.execute(mySql_insert_query)
connection.commit()
print("Record inserted successfully into Laptop table")
cursor.close()
except Error as e:
print("Error while connecting to MysqL", e)
def Register():
Username = input("Username:")
Password = input("Pass:")
VeriEkleme(str(Username),str(Password))
def EnterSystem():
Login = "login"
Answer = input("Login or Register?: ").lower()
if Answer == Login:
print("eşit")
Register()
else:
EnterSystem()
EnterSystem()
Login or Register?: login
eşit
Username:s
Pass:s
MySQL bağlantısı aktif edildi.
Traceback (most recent call last):
File "C:/Users/Relov/PycharmProjects/PygameProje1/PygameProjesi.py", line 36, in <module>
EnterSystem()
File "C:/Users/Relov/PycharmProjects/PygameProje1/PygameProjesi.py", line 32, in EnterSystem
Register()
File "C:/Users/Relov/PycharmProjects/PygameProje1/PygameProjesi.py", line 24, in Register
VeriEkleme(str(Username),str(Password))
File "C:/Users/Relov/PycharmProjects/PygameProje1/PygameProjesi.py", line 11, in VeriEkleme
mySql_insert_query = """INSERT INTO userinformations (Username, Password) VALUES """(Deger1,Deger2)
TypeError: 'str' object is not callable
Process finished with exit code 1
You're calling str as it is a function.
"""INSERT INTO userinformations (Username, Password) VALUES """(Deger1,Deger2)
I recommend using Prepared Statements. This is safe against SQL Injection attacks.
mySql_insert_query = """INSERT INTO userinformations (Username, Password) VALUES (%s, %s)"""
cursor = connection.cursor()
result = cursor.execute(mySql_insert_query, (Deger1, Deger2))
I have a python AWS lambda function that takes JSON records, checks them to see if they have required keys, and then inserts into a MySQL db (AWS RDS Aurora). The function gets invoked whenever a new record comes into the stream def handler.
At the moment, Lambda is reporting some errors, but when I look at cloudwatch logs I don't see any errors, which leads me to believe that maybe I'm not handling or catching the exception. Can anyone tell me where the issue might be?
from __future__ import print_function
import base64
import json
import pymysql
RDS_HOST = 'host'
DB_USER = 'dummy_user'
DB_PASSWORD = 'password1234'
DB_NAME = 'crazy_name'
DB_TABLE = 'wow_table'
class MYSQL(object):
'''
This a wrapper Class for PyMySQL
'''
CONNECTION_TIMEOUT = 30
def __init__(self, host, user, password, database, table):
self.host = host
self.user = user
self.password = password
self.database = database
self.table = table
self.connection = self.connect()
def connect(self):
'''
Connects to MySQL instance
'''
try:
connection = pymysql.connect(
host=self.host,
user=self.user,
password=self.password,
db=self.database,
connect_timeout=self.CONNECTION_TIMEOUT
)
return connection
except Exception as ex:
print(ex)
print("ERROR: Unexpected error: Could not connect to AuroraDB instance")
def execute(self, account_id, external_ref_id, timestamp):
'''
Executes command given a MySQL connection
'''
with self.connection.cursor() as cursor:
sql = ('INSERT INTO ' +
self.database +
'.' +
self.table +
'(`account_id`, `external_reference_id`, `registration`, `c_name`, `c_id`, `create_date`)' +
' VALUES (%s, %s, DATE_FORMAT(STR_TO_DATE(%s,"%%Y-%%M-%%d %%H:%%i:%%s"),"%%Y-%%m-%%d %%H:%%i:%%s"), %s, %s, current_timestamp())' +
' ON DUPLICATE KEY UPDATE create_date = VALUES(create_date)')
cursor.execute(sql, (
account_id,
external_ref_id,
timestamp,
'bingo',
300)
)
self.connection.commit()
def close_connection(self):
'''
Closes connection to MySQL
'''
self.connection.close()
def get_data_from_kinesis_object(obj):
'''
Retrieves data from kinesis event
'''
return obj['kinesis']['data']
def decode_data(data):
'''
Decodes record via base64
'''
return base64.b64decode(data)
def split_records_into_record(records):
'''
Splits a record of records into an array of records
'''
return records.split('\n')
def parse_record(record):
'''
parses record into JSON
'''
if record:
return json.loads(record)
def is_record_valid(record):
'''
Check for keys in event
returns True if they all exist
and False if they dont all exist
'''
return all(key in record for key in (
'eventName',
'sourceType',
'AccountId',
'Timestamp',
'ExternalReferenceId'
))
def handler(event, context):
"""
This function inserts data into Aurora RDS instance
"""
mysql = MYSQL(RDS_HOST, DB_USER, DB_PASSWORD, DB_NAME, DB_TABLE)
for obj in event['Records']:
records = decode_data(get_data_from_kinesis_object(obj))
split_records = split_records_into_record(records)
for record in split_records:
parsed_record = parse_record(record)
if is_record_valid(parsed_record):
mysql.execute(
parsed_record['AccountId'],
parsed_record['ExternalReferenceId'],
str(parsed_record['Timestamp'])
)
mysql.close_connection()
I try to create a function for database connection in python. But this is now working.
Here is my code for the definition.
def connect():
dsn = cx_Oracle.makedsn(host='MYHOST', sid='DEVPRON', port=1521)
conn = cx_Oracle.connect(user='root', password='***', dsn=dsn)
cur = conn.cursor()
return [cur,conn]
I return conn and cur every time i call the connect function.
so here is my code when iam calling the function
connect()[0].execute("insert into tbluser (fullname,nickname) values ('" + fname + "', '" + nname + "') ")
connect()[1].commit()
when i run this no error occur, but when i check the database, there is no inserted row. please help. Thanks
Each time you call your connect function you are creating a new connection to the database server. So, your first call executes a query. The second call gives you a new connection. You're committing with this new connection, but there have been no changes. Try this instead:
def connect():
dsn = cx_Oracle.makedsn(host='MYHOST', sid='DEVPRON', port=1521)
conn = cx_Oracle.connect(user='root', password='***', dsn=dsn)
cur = conn.cursor()
return cur, conn
cur, conn = connect()
cur.execute("insert into tbluser (fullname,nickname) values ('" + fname + "', '" + nname + "') ")
conn.commit()
using sqlalchemy also you can connect database from python. Here is the code
from sqlalchemy import create_engine
engine = create_engine('oracle://host:port/database', echo=True)
conn = engine.connect()
result = conn.execute(query)