Develop Reference

Error closing postgres connection between each function

I have 2 python functions that handle an event in a lambda function that are essentially the same thing. When checking the logs in AWS I get the following error:
{
"errorMessage": "local variable 'post_connection' referenced before assignment",
"errorType": "UnboundLocalError",
"stackTrace": [
" File \"/var/task/etl_python/handler.py\", line 11, in handle\n EtlCall.run_bc(event)\n",
" File \"/var/task/etl_python/elt_call.py\", line 153, in run_bc\n if post_connection:\n"
]
}
My code looks like this:
def run_bo(event):
s3_resource = boto3.resource('s3')
idv_endpoint = os.getenv('DB_ENDPOINT')
idv_database = os.getenv("DB_NAME")
filename = 'staging/aml_bo'
bucket = os.getenv('BILLING_ETL')
if 'resources' in event and "psql_billing" in event['resources'][0]:
try:
config = VaultService()
s3_resource = boto3.resource('s3')
idv_endpoint = os.getenv('DB_ENDPOINT')
idv_database = os.getenv("DB_NAME")
filename = 'staging/billing_bo'
bucket = os.getenv('BILLING_ETL')
idv_username = config.get_postgres_username()
idv_password = config.get_postgres_password()
post_connection = psycopg2.connect(user = idv_username
, password = idv_password
, host = idv_endpoint
, port = "5432"
, database = idv_database)
cursor = post_connection.cursor()
bo_qry = "SELECT uuid\
,first_name, middle_initial, last_name, date(date_of_birth)
mailing_address, public.bo"
#Might need not need the next two lines but this should work.
query = """COPY ({}) TO STDIN WITH (FORMAT csv, DELIMITER '|', QUOTE '"', HEADER TRUE)""".format(bo_qry)
file = StringIO()
cursor.copy_expert(query, file)
s3_resource.Object(bucket, f'{filename}.csv').put(Body=file.getvalue())
cursor.close()
except(Exception, psycopg2.Error) as error:
print("Error connecting to postgres instance", error)
finally:
if post_connection:
cursor.close()
post_connection.close()
#return "SUCCESS"
else:
# Unknown notification
#raise Exception(f'Unexpected event notification: {event}')
print("Cannot make a solid connection to psql instance. Please check code configuration")
def run_bc(event):
if 'resources' in event and "psql_billing" in event['resources'][0]:
try:
config = VaultService()
s3_resource = boto3.resource('s3')
idv_endpoint = os.getenv('DB_ENDPOINT')
idv_database = os.getenv("DB_NAME")
filename = 'staging/billing_bc'
bucket = os.getenv('BILLING_ETL')
idv_username = config.get_postgres_username()
idv_password = config.get_postgres_password()
post_connection = psycopg2.connect(user = idv_username
, password = idv_password
, host = idv_endpoint
, port = "5432"
, database = idv_database)
cursor = post_connection.cursor()
bc_qry = "select id, uuid, document_type, image_id,
document_id\
from public.bc"
#Might need not need the next two lines but this should work.
query = """COPY ({}) TO STDIN WITH (FORMAT csv, DELIMITER '|', QUOTE '"', HEADER TRUE)""".format(bc_flowdown_qry)
file = StringIO()
cursor.copy_expert(query, file)
s3_resource.Object(bucket, f'{filename}.csv').put(Body=file.getvalue())
cursor.close()
except(Exception, psycopg2.Error) as error:
print("Error connecting to postgres instance", error)
finally:
if post_connection:
cursor.close()
post_connection.close()
#return "SUCCESS"
else:
# Unknown notification
#raise Exception(f'Unexpected event notification: {event}')
print("Cannot make a solid connection to psql instance. Please check code configuration")
I don't understand how my connection is unbound if I am closing the connection and the connection after each function and then reopening it for the next. I close it at the end when the data is dumped to my file and then create a new connection in the next function.

syntax error in mariaDB that i cannot find

i am writing a code to write my gps variables in my database but i get this error everytime i run it:
You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'Long, Status) VALUES (1, 50.821139333333335, 3.2815086666666664, 'Eind')' at line 1
the database has an id that is auto incremented and then the following values i'm trying to add, no clue what's going wrong. maybe you guys have an idea
database.py:
#staticmethod
def execute_sql(sqlQuery, params=None):
result = None
db, cursor = Database.__open_connection()
try:
cursor.execute(sqlQuery, params)
db.commit()
result = cursor.lastrowid
except connector.Error as error:
db.rollback()
result = None
print(f"Error: Data not stored.{error.msg}")
finally:
cursor.close()
db.close()
return result
datarepository.py:
#staticmethod
def create_gps(RouteID, Lat, Long, Status):
sql = "INSERT INTO gps (RouteID, Lat, Long, Status) VALUES (%s, %s, %s, %s)"
params = [RouteID, Lat, Long, Status]
return Database.execute_sql(sql, params)
app.py:
#socketio.on('F2B_GPS')
def gps_aan(toggle, status):
while(toggle != 1):
port = "/dev/serial0"
ser = serial.Serial(port, baudrate=9600, timeout=0.5)
dataout = pynmea2.NMEAStreamReader()
newdata = ser.readline()
if sys.version_info[0] == 3:
newdata = newdata.decode("utf-8", "ignore")
if newdata[0:6] == "$GPRMC":
newmsg = pynmea2.parse(newdata)
lat = newmsg.latitude
lng = newmsg.longitude
gps = "Latitude=" + str(lat) + "and Longitude=" + str(lng)
print(gps)
DataRepository.create_gps(1,lat,lng, status)
print("added")
toggle = 1
Thanks to anyone replying :)

ok i found it you cant use Long since that screws it up in mysql so i had to change it to longitude

How to add exception to python function?

Good day. I wrote an application on python that collects call logs from the Avaya PBX and writes them to the mysql database. It works well, but sometimes the PBX sends an empty string for some reason and the program fails. I attach the screen and code below. I understand that you need to wrap the function in an exception: try except, but I don’t understand how to do it. Please tell me how to do this.enter image description here
def write_db(item, *agrs):
connection = pymysql.connect(host='localhost',
user='acdr',
password='it8ejokd',
db='avaya_cdr',
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor)
DBTBL = "cdr102019"
DBFLD = "Date_call, Time_call, `Sec_dur`, `clg_num_in_tag`, `dialed_num`, dep_called, dep_dialed"
dep_num_call = find_dep(item[3].replace(' ', ''))
name_dep_call = name_dep(dep_num_call)
dep_num_dial = find_dep(item[4].replace(' ', ''))
name_dep_dial = name_dep(dep_num_dial)
item.append(name_dep_call)
item.append(name_dep_dial)
item[1] = item[1] + "00"
try:
with connection.cursor() as cursor:
sql = "INSERT INTO "+DBTBL+" ("+DBFLD+") VALUES (%s,%s,%s,%s,%s,%s,%s)"
cursor.execute(sql, (item))
connection.commit()
finally:
connection.close()
# Задаем адрес сервера
SERVER_ADDRESS = ('', 5100)
# Настраиваем сокет
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.bind(SERVER_ADDRESS)
server_socket.listen(5)
print('server is running, please, press ctrl+c to stop')
# Слушаем запросы и пишем в db
while True:
connection, address = server_socket.accept()
data = connection.recv(1024)
if not(b'\x00\x00\x00' in data) and not(b'1370' in data):
str = data.decode("utf-8")
item=[str[0:6],str[7:11],str[12:17],str[18:33],str[34:57]]
print(item)
write_db(item)
connection.close()

You'll have to catch the exception, so we can cater for a few types, but just to be sure and get you up and running, you could do the following :)
try:
with connection.cursor() as cursor:
sql = (
"INSERT INTO "+DBTBL+" ("+DBFLD+") VALUES "
"(%s,%s,%s,%s,%s,%s,%s)"
)
cursor.execute(
sql,
(item),
)
connection.commit()
except Exception as e:
print("Error occurred: %s"% e)
finally:
connection.close()

This should do the trick. I've used all four elements of try/except/else/finally here, with brief explanations of when they're executed.
try:
with connection.cursor() as cursor:
sql = "INSERT INTO "+DBTBL+" ("+DBFLD+") VALUES (%s,%s,%s,%s,%s,%s,%s)"
cursor.execute(sql, (item))
except Exception: # If this code fails, ignore it
pass
else: # If the code inside 'try' succeeds, execute this code.
connection.commit()
finally: # Regardless of whether or not the code inside 'try' succeeds, execute this code
connection.close()

Nmap scan using python and store in mysql

python code using nmap to scan a range of ip's and then store those ip's individually in mysql table.
how to scan when we need to scan a range of ip, e.g. 172.27.20.130-140. and store all ips in table of mysql. now a range is needed to be searched and each ip should be stored in table.
#NMAP SCANNING CODE
nmScan=nmap.PortScanner()
host=self._firstname.value # taking input from a pyforms gui input field
result=nmScan.scan(hosts=host, arguments='-sV -v -p 1-1024')
print('Host : %s (%s)' % (host, nmScan[host].hostname()))
print('State : %s' % nmScan[host].state())
for proto in nmScan[host].all_protocols():
print('----------')
print('Protocol : %s' % proto)
lport = nmScan[host][proto].keys()
#lport.sort()
for port in lport:
thisDict = nmScan[host][proto][port]
print ('port : %s\tstate : %s\tVersion:%s,v%s'% (port, nmScan[host][proto][port]['state'],thisDict['product'] ,thisDict['version']))
#INSERT into INPUT table CODE
try:
connection = mysql.connector.connect(host='localhost',
database='testdb',
user='root',
password='Pooja#123')
cursor = connection.cursor()
sql_insert_query = """ INSERT INTO input (sno,ip) VALUES (%s,%s)"""
so=cursor.lastrowid
sno=so
ip=self._firstname.value # taking input from a pyforms gui input field
insert_tuple = (sno,ip)
result = cursor.execute(sql_insert_query, insert_tuple)
print("inserted")
connection.commit()
except mysql.connector.Error as error :
connection.rollback() #rollback if any exception occured
print("Failed inserting record into input table {}".format(error))
finally:
#closing database connection.
if(connection.is_connected()):
cursor.close()
connection.close()
print("MySQL connection is closed")

The ipaddress module is useful for this type of thing:
In [1]: import ipaddress
In [2]: addr = ipaddress.IPv4Address('172.27.20.130')
In [3]: print(addr)
172.27.20.130
In [4]: print(addr + 1)
172.27.20.131

PyMySQL not Inserting Data

I have a piece of code which is taking Windows logs and inserting various pieces of information into an mySQL database. The code is running perfectly with no errors, but does not actually input the data into the table. The table remains blank. I pulled my mySQL syntax from an example with some modification, so I'm not entirely sure what is going wrong. I have a feeling it has either to do with the data types, or some changes I made to the syntax.
import sys
import pymysql
import pymysql.cursors
import win32evtlog # requires pywin32 pre-installed
import win32evtlogutil
import time
server = 'localhost' # name of the target computer to get event logs
logtype = 'System' # 'Application' # 'Security'
hand = win32evtlog.OpenEventLog(server,logtype)
flags =
win32evtlog.EVENTLOG_BACKWARDS_READ|win32evtlog.EVENTLOG_SEQUENTIAL_READ
events = win32evtlog.ReadEventLog(hand, flags,0)
while True:
for event in events:
evt_tp = event.EventType
if evt_tp != (1 or 2 or 8):
eve_cat = str(('Event Category:', event.EventCategory))
eve_timegen = str(('Time Generated:', event.TimeGenerated))
eve_srcnm = str(('Source Name:', event.SourceName))
eve_id = str(('Event ID:', event.EventID))
eve_typ = str(('Event Type:', event.EventType))
data = event.StringInserts
if data:
print ('Event Data:')
for msg in data:
print(msg)
print(type(eve_cat))
print(type(eve_timegen))
print(type(eve_srcnm))
print(type(eve_id))
print(type(eve_typ))
print(type(data))
time.sleep(10)
else:
eve_cat = ('Event Category:', event.EventCategory)
eve_timegen = ('Time Generated:', event.TimeGenerated)
eve_srcnm = ('Source Name:', event.SourceName)
eve_id = ('Event ID:', event.EventID)
eve_typ = ('Event Type:', event.EventType)
data = event.StringInserts
print('There were no errors found')
print(eve_cat)
print(eve_timegen)
print(eve_srcnm)
print(eve_id)
print(eve_typ)
print(data)
time.sleep(10)
# Connect to the database
connection = pymysql.connect(host='localhost',
user='root',
password='',
db='ptest',
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor)
try:
with connection.cursor() as cursor:
# Create a new record
sql = "INSERT INTO `win_logs` (`Category`, `TimeGenerated`, 'SourceName',
'EventID', 'Type') VALUES (%s, %s, %s, %s, %s)"
cursor.execute(sql, (eve_cat, eve_timegen, eve_srcnm, eve_id, eve_typ))
# connection is not autocommit by default. So you must commit to save
# your changes.
connection.commit()
with connection.cursor() as cursor:
# Read a single record
sql = "SELECT `id`, `Type` FROM `win_logs` WHERE `Category`=%s"
cursor.execute(sql, ('webmaster#python.org',))
result = cursor.fetchone()
print(result)
finally:
connection.close()

I can be very wrong.
But this is python.
Indentation matter.
Try just:
try:
with connection.cursor() as cursor:
# Create a new record
sql = "INSERT INTO `win_logs` (`Category`, `TimeGenerated`, 'SourceName`, 'EventID', 'Type') VALUES (%s, %s, %s, %s, %s)"
cursor.execute(sql, (eve_cat, eve_timegen, eve_srcnm, eve_id, eve_typ))
I guess your cursor is out of with scope