Develop Reference

Best Practice: looking to structure my aws lambda function with pandas manipulations

Im new to AWS Lambda so please take it easy on me :)
Im getting a lot of errors in my code and Im not sure the best way to troubleshoot other than look at the cloudwatch console and adjust things as necessary. If anyone has any tips for troubleshooting Id appreciate it!
Heres my plan for what I want to do and please let me know if this makes sense:
upload file to s3 bucket -> 2. upload triggers a lambda to run (inside this lambda is a python script that modifies the data source. The source data is a messy file) -> 3. store the output to the same s3 bucket in a separate folder - > 4. (future state) perform analysis on the new json file.
I have my s3 bucket created and I have setup the lambda to trigger when a new file is added. That part is working! I have added my python script (which works on my local drive) portion to the lambda function w/in the code section of lambda.
The errors am getting errors consist of saying that my 6 global variables (df_a1-df_aq) are not defined. If I move them out of the function then it works, however when I get to the merge portion of my code I am getting an error saying that says "cannot merge a series without a name" I gave them a name using the name= object and Im still getting this issue.
Here's my code that is in my aws lambda:
try:
import json
import boto3
import pandas as pd
import time
import io
print("All Modules are ok ...")
except Exception as e:
print("Error in Imports ")
s3_client = boto3.client('s3')
#df_a1 = pd.Series(dtype='object', name='test1')
#df_g1 = pd.Series(dtype='object', name='test2')
#df_j1 = pd.Series(dtype='object', name='test3')
#df_p1 = pd.Series(dtype='object', name='test4')
#df_r1 = pd.Series(dtype='object', name='test5')
#df_q1 = pd.Series(dtype='object', name='test6')
def Add_A1 (xyz, RC, string):
#DATA TO GRAB FROM STRING
global df_a1
IMG = boolCodeReturn(string[68:69].strip())
roa = string[71:73].strip()
#xyzName = string[71:73].strip()
#ADD RECORD TO DATAFRAME
series = pd.Series (data=[xyz, IMG, roa], index=['XYZ', 'IMG', 'Roa'])
df_a1 = df_a1.append(series, ignore_index=True)
def Add_G1 (xyz, RC, string):
global df_g1
#DATA TO GRAB FROM STRING
gcode = string[16:30].strip()
ggname = string[35:95].strip()
#ADD RECORD TO DATAFRAME
series = pd.Series (data=[xyz, gcode, ggname], index=['XYZ', 'Gcode', 'Ggname'])
df_g1 = df_g1.append(series, ignore_index=True)
def Add_J1 (xyz, RC, string):
#DATA TO GRAB FROM STRING
global df_j1
xyzName = string[56:81].strip()
#ADD RECORD TO DATAFRAME
series = pd.Series (data=[xyz, xyzName], index=['XYZ', 'XYZName'])
df_j1 = df_j1.append(series, ignore_index=True)
def Add_P01 (xyz, RC, string):
global df_p1
#DATA TO GRAB FROM STRING
giname = string[50:90].strip()
#ADD RECORD TO DATAFRAME
series = pd.Series (data=[xyz, giname], index=['XYZ', 'Giname'])
df_p1 = df_p1.append(series, ignore_index=True)
def Add_R01 (xyz, RC, string):
global df_r1
#DATA TO GRAB FROM STRING
Awperr = boolCodeReturn(string[16:17].strip())
#PPP= string[17:27].lstrip("0")
AUPO = int(string[27:40].lstrip("0"))
AUPO = AUPO / 100000
AupoED = string[40:48]
#ADD RECORD TO DATAFRAME
series = pd.Series (data=[xyz, AUPO, Awperr, AupoED], index = ['XYZ', 'AUPO', 'Awperr', 'AupoED'])
df_r1 = df_r1.append(series, ignore_index=True)
def Add_Q01 (xyz, RC, string):
global df_q1
#DATA TO GRAB FROM STRING
#PPP= string[17:27].lstrip("0")
UPPWA = int(string[27:40].lstrip("0"))
UPPWA = UPPWA / 100000
EDWAPPP = string[40:48]
#ADD RECORD TO DATAFRAME
series = pd.Series (data=[xyz, UPPWA, EDWAPPP], index = ['XYZ', 'UPPWA', 'EDWAPPPer'])
df_q1 = df_q1.append(series, ignore_index=True)
def boolCodeReturn (code):
if code == "X":
return 1
else:
return 0
def errorHandler(xyz, RC, string):
pass
def lambda_handler(event, context):
print(event)
#Get Bucket Name
bucket = event['Records'][0]['s3']['bucket']['name']
#get the file/key name
key = event['Records'][0]['s3']['object']['key']
response = s3_client.get_object(Bucket=bucket, Key=key)
print("Got Bucket! - pass")
print("Got Name! - pass ")
data = response['Body'].read().decode('utf-8')
print('reading data')
buf = io.StringIO(data)
print(buf.readline())
#data is the file uploaded
fileRow = buf.readline()
print('reading_row')
while fileRow:
currentString = fileRow
xyz = currentString[0:11].strip()
RC = currentString[12:15].strip() #this grabs the code the indicates what the data type is
#controls which function to run based on the code
switcher = {
"A1": Add_A1,
"J1": Add_J1,
"G1": Add_G1,
"P01": Add_P01,
"R01": Add_R01,
"Q01": Add_Q01
}
runfunc = switcher.get(RC, errorHandler)
runfunc (xyz, RC, currentString)
fileRow = buf.readline()
print(type(df_a1), "A1 FILE")
print(type(df_g1), 'G1 FILE')
buf.close()
##########STEP 3: JOIN THE DATA TOGETHER##########
df_merge = pd.merge(df_a1, df_g1, how="left", on="XYZ")
df_merge = pd.merge(df_merge, df_j1, how="left", on="XYZ")
df_merge = pd.merge(df_merge, df_p1, how="left", on="XYZ")
df_merge = pd.merge(df_merge, df_q1, how="left", on="XYZ")
df_merge = pd.merge(df_merge, df_r1, how="left", on="XYZ")
##########STEP 4: SAVE THE DATASET TO A JSON FILE##########
filename = 'Export-Records.json'
json_buffer = io.StringIO()
df_merge.to_json(json_buffer)
s3_client.put_object(Buket='file-etl',Key=filename, Body=json_buffer.getvalue())
t = time.localtime()
current_time = time.strftime("%H:%M:%S", t)
print("Finished processing at " + current_time)
response = {
"statusCode": 200,
'body': json.dumps("Code worked!")
}
return response
Here are some of the error messages:
[ERROR] NameError: name 'df_a1' is not defined
Traceback (most recent call last):
File "/var/task/lambda_function.py", line 145, in lambda_handler
runfunc (ndc, recordcode, currentString)
File "/var/task/lambda_function.py", line 26, in Add_A1
df_a1 = df_a1.append(series, ignore_index=True)
[ERROR] NameError: name 'df_g1' is not defined
Traceback (most recent call last):
File "/var/task/lambda_function.py", line 152, in lambda_handler
runfunc (ndc, recordcode, currentString)
File "/var/task/lambda_function.py", line 38, in Add_G1
df_g1 = df_g1.append(series, ignore_index=True)
[ERROR] ValueError: Cannot merge a Series without a name
Traceback (most recent call last):
File "/var/task/lambda_function.py", line 160, in lambda_handler
df_merge = pd.merge(df_a1, df_g1, how="left", on="NDC")
File "/opt/python/pandas/core/reshape/merge.py", line 111, in merge
op = _MergeOperation(
File "/opt/python/pandas/core/reshape/merge.py", line 645, in __init__
_left = _validate_operand(left)
File "/opt/python/pandas/core/reshape/merge.py", line 2425, in _validate_operand
raise ValueError("Cannot merge a Series without a name")

Python fernet: token must be in bytes

i would like to add values to the database in sqlite3, and they need to be encrypted. And when i want to retrieve values they need to be decrypted. Now I'm getting this error TypeError: token must be bytes. Here is the full list of errors: Traceback (most recent call last): File "C:\Users\d\OneDrive\Bureaublad\Assignment8\CDMS.py", line 75, in <module> get_clients() File "C:\Users\d\OneDrive\Bureaublad\Assignment8\CDMS.py", line 68, in get_clients new += fernet.decrypt(i).decode() + " " File "C:\Users\d\AppData\Local\Programs\Python\Python38\lib\site-packages\cryptography\fernet.py", line 75, in decrypt timestamp, data = Fernet._get_unverified_token_data(token) File "C:\Users\d\AppData\Local\Programs\Python\Python38\lib\site-packages\cryptography\fernet.py", line 100, in _get_unverified_token_data utils._check_bytes("token", token) File "C:\Users\d\AppData\Local\Programs\Python\Python38\lib\site-packages\cryptography\utils.py", line 29, in _check_bytes raise TypeError("{} must be bytes".format(name)) TypeError: token must be bytes
Here is my code:
from Classes import Client
from cryptography.fernet import Fernet
import sqlite3
key = Fernet.generate_key()
fernet = Fernet(key)
conn = sqlite3.connect('database.db')
c = conn.cursor()
def insert_client(client):
fullname = fernet.encrypt(client.fullname.encode())
fullname1 = fullname.decode()
adress = fernet.encrypt(client.adress.encode())
adress1 = adress.decode()
zipcode = fernet.encrypt(client.zipcode.encode())
zipcode1 = zipcode.decode()
city = fernet.encrypt(client.city.encode())
city1 = city.decode()
email = fernet.encrypt(client.email.encode())
email1 = email.decode()
mphone = fernet.encrypt(client.mphone.encode())
mphone1 = mphone.decode()
with conn:
c.execute("INSERT INTO client VALUES (:fullname, :adress, :zipcode, :city, :email, :mphone)",
{'fullname': fullname1, 'adress': adress1, 'zipcode': zipcode1,
'city': city1, 'email': email1, 'mphone': mphone1})
client1 = Client('Name1', 'Street1', 'Zipcode', 'NewYork', '123#gmail.com', '12345678')
def get_clients():
arr = []
new = ""
c.execute("SELECT * FROM client")
arr = c.fetchall()
for i in arr:
new += fernet.decrypt(i).decode() + " "
return new
insert_client(client1)
get_clients()
conn.close()
The first function is for adding clients, by a class object named Client. And the second function is for retrieving all values decrypted.

fetchall returns an iterable of rows (most commonly tuples). If you want to only process one record, you should instead use fetchone:
...
arr = c.fetchone()
for i in arr:
new += fernet.decrypt(i).decode() + " "
...
Alternatively you could iterate the cursor and return a list:
def get_clients():
arr = []
c.execute("SELECT * FROM client")
for row in c:
new = ""
for i in row:
new += fernet.decrypt(i).decode() + " "
arr.append(new)
return arr
But the Pythonic way would be to use comprehensions:
def get_clients():
c.execute("SELECT * FROM client")
return [' '.join(fernet.decrypt(i).decode() for i in row)
for row in c]
Slightly shorter, isn't it?

pyodbc ERROR - ('ODBC SQL type -151 is not yet supported. column-index=16 type=-151', 'HY106')

I'm working on automating some query extraction using python and pyodbc, and then converting to parquet format, and send to AWS S3.
My script solution is working fine so far, but I have faced a problem. I have a Schema, let us call it SCHEMA_A, and inside of it several tables, TABLE_1, TABLE_2 .... TABLE_N.
All those tables inside that schema are accessible by using the same credentials.
So I'm using a script like this one to automate the task.
def get_stream(cursor, batch_size=100000):
while True:
row = cursor.fetchmany(batch_size)
if row is None or not row:
break
yield row
cnxn = pyodbc.connect(driver='pyodbc driver here',
host='host name',
database='schema name',
user='user name,
password='password')
print('Connection stabilished ...')
cursor = cnxn.cursor()
print('Initializing cursos ...')
if len(sys.argv) > 1:
table_name = sys.argv[1]
cursor.execute('SELECT * FROM {}'.format(table_name))
else:
exit()
print('Query fetched ...')
row_batch = get_stream(cursor)
print('Getting Iterator ...')
cols = cursor.description
cols = [col[0] for col in cols]
print('Initalizin batch data frame ..')
df = pd.DataFrame(columns=cols)
start_time = time.time()
for rows in row_batch:
tmp = pd.DataFrame.from_records(rows, columns=cols)
df = df.append(tmp, ignore_index=True)
tmp = None
print("--- Batch inserted inn%s seconds ---" % (time.time() - start_time))
start_time = time.time()
I run a code similar to that inside Airflow tasks, and works just fine for all other tables. But then I have two tables, lets call TABLE_I and TABLE_II that yields the following error when I execute cursor.fetchmany(batch_size):
ERROR - ('ODBC SQL type -151 is not yet supported. column-index=16 type=-151', 'HY106')
Traceback (most recent call last):
File "/home/ubuntu/.local/lib/python3.8/site-packages/airflow/models/taskinstance.py", line 1112, in _run_raw_task
self._prepare_and_execute_task_with_callbacks(context, task)
File "/home/ubuntu/.local/lib/python3.8/site-packages/airflow/models/taskinstance.py", line 1285, in _prepare_and_execute_task_with_callbacks
result = self._execute_task(context, task_copy)
File "/home/ubuntu/.local/lib/python3.8/site-packages/airflow/models/taskinstance.py", line 1310, in _execute_task
result = task_copy.execute(context=context)
File "/home/ubuntu/.local/lib/python3.8/site-packages/airflow/operators/python.py", line 117, in execute
return_value = self.execute_callable()
File "/home/ubuntu/.local/lib/python3.8/site-packages/airflow/operators/python.py", line 128, in execute_callable
return self.python_callable(*self.op_args, **self.op_kwargs)
File "/home/ubuntu/prea-ninja-airflow/jobs/plugins/extract/fetch.py", line 58, in fetch_data
for rows in row_batch:
File "/home/ubuntu/prea-ninja-airflow/jobs/plugins/extract/fetch.py", line 27, in stream
row = cursor.fetchmany(batch_size)
Inspecting those tables with SQLElectron, and Querying the first few lines, I have realized that both TABLE_I and TABLE_II have a Column called 'Geolocalizacao', when I use SQL server language to find the DATA TYPE of that column with:
SELECT DATA_TYPE
FROM INFORMATION_SCHEMA.COLUMNS
WHERE
TABLE_NAME = 'TABLE_I' AND
COLUMN_NAME = 'Geolocalizacao';
It yields:
DATA_TYPE
geography
Seraching here on stack overflow I found this solution: python pyodbc SQL Server Native Client 11.0 cannot return geometry column
By the description of the user, it seem work fine by adding:
def unpack_geometry(raw_bytes):
# adapted from SSCLRT information at
# https://learn.microsoft.com/en-us/openspecs/sql_server_protocols/ms-ssclrt/dc988cb6-4812-4ec6-91cd-cce329f6ecda
tup = struct.unpack('<i2b3d', raw_bytes)
# tup contains: (unknown, Version, Serialization_Properties, X, Y, SRID)
return tup[3], tup[4], tup[5]
and then:
cnxn.add_output_converter(-151, unpack_geometry)
After creating the connection. But It's not working for the GEOGRAPHY DATA TYPE, when I use this code (add import struct on python script), it gives me the following error:
Traceback (most recent call last):
File "benchmark.py", line 79, in <module>
for rows in row_batch:
File "benchmark.py", line 39, in get_stream
row = cursor.fetchmany(batch_size)
File "benchmark.py", line 47, in unpack_geometry
tup = struct.unpack('<i2b3d', raw_bytes)
struct.error: unpack requires a buffer of 30 bytes
An example of values that this column have, follows the given template:
{"srid":4326,"version":1,"points":[{}],"figures":[{"attribute":1,"pointOffset":0}],"shapes":[{"parentOffset":-1,"figureOffset":0,"type":1}],"segments":[]}
I honestly don't know how to adapt the code for this given structure, can someone help me? It's been working fine for all other tables, but I have those two tables with this column that are giving me a lot o headeach.

Hi this is what I have done:
from binascii import hexlify
def _handle_geometry(geometry_value):
return f"0x{hexlify(geometry_value).decode().upper()}"
and then on connection:
cnxn.add_output_converter(-151, _handle_geometry)
this will return value as SSMS.

Python script makes AssertionError when trying to MariaDB

I have this piece of code that collects data from a HAT connected to a Raspberry.
When run it gives this error:
[51.57, 22.30, 1002.01]
Traceback (most recent call last):
File "dbWriter.py", line 45, in <module>
write2DB(record)
File "dbWriter.py", line 26, in write2DB
assert len(values) == 3
AssertionError
I am by no means a programmer, i just fiddle around. It is meant to save 'record' to a database, which is then read and updated in realtime on an apache2 server. All help appreciated.
import mysql.connector
from itertools import repeat
import sys
import bme680
import time
try:
sensor = bme680.BME680(bme680.I2C_ADDR_PRIMARY)
except IOError:
sensor = bme680.BME680(bme680.I2C_ADDR_SECONDARY)
sensor.set_humidity_oversample(bme680.OS_2X)
sensor.set_pressure_oversample(bme680.OS_4X)
sensor.set_temperature_oversample(bme680.OS_8X)
sensor.set_filter(bme680.FILTER_SIZE_3)
mydb = mysql.connector.connect(
host='localhost',
user='pi',
passwd='pass',
database='weatherDB'
)
mycursor = mydb.cursor()
def write2DB(values):
assert len(values) == 3
sqlText = '''INSERT INTO data(humidity, temperature, pressure) VALUES({},{}, })'''.format(values[0], values[1], values[2])
mycursor.execute(sqlText)
mydb.commit()
for _ in repeat(None):
sensor.get_sensor_data()
output_humi = '{0:.2f}'.format(
sensor.data.humidity)
output_temp = '{0:.2f}'.format(
sensor.data.temperature)
output_pres = '{0:.2f}'.format(
sensor.data.pressure)
record = []
record = ('[' + (output_humi) + ', ' + (output_temp) + ', ' + (output_pres) + ']')
print(record)
write2DB(record)
time.sleep(10)
pass

You have:
record = ('[' + (output_humi) + ', ' + (output_temp) + ', ' + (output_pres) + ']')
record evaluates to a single string, not a list of 3 elements and hence your exception.
Change the above to:
record = [output_humi, output_temp, output_pres]
You are also missing a { in your format specification. It should be:
sqlText = '''INSERT INTO data(humidity, temperature, pressure) VALUES({},{}, {})'''.format(values[0], values[1], values[2])
An alternative would be to use a prepared statement:
sqlText = 'INSERT INTO data(humidity, temperature, pressure) VALUES(%s, %s, %s)'
mycursor.execute(sqlText, values)
In the above case you will be passing actual strings as the values. I don't know how the columns are defined, but no matter. If they are defined as floating point or decimal values, the strings will be converted to the correct type.

Peewee raw SELECT sql with multiple programmatic filter parameters

I am facing a problem returning the users from this get_users() function. Here is the code:
I am using Peewee, Pymysql and MySQL
def get_users(self, filter_columns=None, parameters=[], operator=None, ):
#Define the operator to be used in the WHERE statement
if operator:
operator = operator
else:
operator = '&'
#Contruct the sql_where statement
where = ''
if filter_columns:
for field in filter_columns:
where = where + field + '=%s, '
if len(where)>2:
where = where[:-2]
#Build Parameter list
param_list = ''
for param in parameters:
param_list = param_list + param + ", "
if len(param_list)>2:
param_list = param_list[:-2]
#Select the users and return.
sql = "SELECT * FROM user WHERE " + where
user = U.raw(sql, param_list)
return user
When i call the function like this:
users = user.get_users(filter_columns=['first_name', 'status'], parameters=['awa', 'active'], operator='|')
print(users)
for u in users:
print(u.first_name, u.last_name)
This is what i get as result:
Traceback (most recent call last):
File "D:/projects/micro bank/tests/smanager/randomtest.py", line 10, in <module>
for u in users:
File "C:\Python34\lib\site-packages\peewee.py", line 2963, in __iter__
return iter(self.execute())
File "C:\Python34\lib\site-packages\peewee.py", line 2959, in execute
self._qr = QRW(self.model_class, self._execute(), None)
File "C:\Python34\lib\site-packages\peewee.py", line 2902, in _execute
return self.database.execute_sql(sql, params, self.require_commit)
File "C:\Python34\lib\site-packages\peewee.py", line 3758, in execute_sql
cursor.execute(sql, params or ())
File "C:\Python34\lib\site-packages\pymysql\cursors.py", line 164, in execute
query = self.mogrify(query, args)
File "C:\Python34\lib\site-packages\pymysql\cursors.py", line 143, in mogrify
query = query % self._escape_args(args, conn)
TypeError: not enough arguments for format string
When i print out the user returned, i get this:
<class 'common.models.User'> SELECT * FROM user WHERE first_name=%s, status=%s ['awa, active']
From observation, the problem comes from this last area ['awa, active'] which is supposed to be ['awa', 'active']
The problem now is establishing a parameter_list that when i use it, it should print out like this ['awa', 'active']
Thanks for assistance.

I assume you need param_list to be ['awa', 'active']
Try this:
param_list = []
for param in parameters:
param_list.append(param)
# Since `param_list` in needed in the format `'"awa", "active"'`
param_list = ', '.join('"{0}"'.format(w) for w in param_list)
instead of
param_list = ''
for param in parameters:
param_list = param_list + param + ", "