I have a piece of code which is taking Windows logs and inserting various pieces of information into an mySQL database. The code is running perfectly with no errors, but does not actually input the data into the table. The table remains blank. I pulled my mySQL syntax from an example with some modification, so I'm not entirely sure what is going wrong. I have a feeling it has either to do with the data types, or some changes I made to the syntax.
import sys
import pymysql
import pymysql.cursors
import win32evtlog # requires pywin32 pre-installed
import win32evtlogutil
import time
server = 'localhost' # name of the target computer to get event logs
logtype = 'System' # 'Application' # 'Security'
hand = win32evtlog.OpenEventLog(server,logtype)
flags =
win32evtlog.EVENTLOG_BACKWARDS_READ|win32evtlog.EVENTLOG_SEQUENTIAL_READ
events = win32evtlog.ReadEventLog(hand, flags,0)
while True:
for event in events:
evt_tp = event.EventType
if evt_tp != (1 or 2 or 8):
eve_cat = str(('Event Category:', event.EventCategory))
eve_timegen = str(('Time Generated:', event.TimeGenerated))
eve_srcnm = str(('Source Name:', event.SourceName))
eve_id = str(('Event ID:', event.EventID))
eve_typ = str(('Event Type:', event.EventType))
data = event.StringInserts
if data:
print ('Event Data:')
for msg in data:
print(msg)
print(type(eve_cat))
print(type(eve_timegen))
print(type(eve_srcnm))
print(type(eve_id))
print(type(eve_typ))
print(type(data))
time.sleep(10)
else:
eve_cat = ('Event Category:', event.EventCategory)
eve_timegen = ('Time Generated:', event.TimeGenerated)
eve_srcnm = ('Source Name:', event.SourceName)
eve_id = ('Event ID:', event.EventID)
eve_typ = ('Event Type:', event.EventType)
data = event.StringInserts
print('There were no errors found')
print(eve_cat)
print(eve_timegen)
print(eve_srcnm)
print(eve_id)
print(eve_typ)
print(data)
time.sleep(10)
# Connect to the database
connection = pymysql.connect(host='localhost',
user='root',
password='',
db='ptest',
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor)
try:
with connection.cursor() as cursor:
# Create a new record
sql = "INSERT INTO `win_logs` (`Category`, `TimeGenerated`, 'SourceName',
'EventID', 'Type') VALUES (%s, %s, %s, %s, %s)"
cursor.execute(sql, (eve_cat, eve_timegen, eve_srcnm, eve_id, eve_typ))
# connection is not autocommit by default. So you must commit to save
# your changes.
connection.commit()
with connection.cursor() as cursor:
# Read a single record
sql = "SELECT `id`, `Type` FROM `win_logs` WHERE `Category`=%s"
cursor.execute(sql, ('webmaster#python.org',))
result = cursor.fetchone()
print(result)
finally:
connection.close()
I can be very wrong.
But this is python.
Indentation matter.
Try just:
try:
with connection.cursor() as cursor:
# Create a new record
sql = "INSERT INTO `win_logs` (`Category`, `TimeGenerated`, 'SourceName`, 'EventID', 'Type') VALUES (%s, %s, %s, %s, %s)"
cursor.execute(sql, (eve_cat, eve_timegen, eve_srcnm, eve_id, eve_typ))
I guess your cursor is out of with scope
Related
I successfully access the database, however, I can't load the table inside the database. I am quite sure that the name of the table is correct, the database is a mimic iii database. Please give me a helping hand, thanks a lot!
import psycopg2
try:
connection = psycopg2.connect(user="postgres",
password="xxxxxxx",
host="localhost",
port="5432",
database="mimic")
cursor = connection.cursor()
postgreSQL_select_Query = "select * from admissions"
cursor.execute(postgreSQL_select_Query)
print("Selecting rows from mobile table using cursor.fetchall")
admissions_records = cursor.fetchall()
print("Print each row and it's columns values")
for row in admissions_records:
print("x = ", row[0], )
print("y = ", row[1])
print("z = ", row[2], "\n")
except (Exception, psycopg2.Error) as error:
print("Error while fetching data from PostgreSQL", error)
finally:
# closing database connection.
if connection:
cursor.close()
connection.close()
print("PostgreSQL connection is closed")
Here's the output:
Error while fetching data from PostgreSQL relation "admissions" does not exist
LINE 1: select * from admissions
^
PostgreSQL connection is closed
i'm studying about mysql connection with python(pycharm)
i have question about curs.execute()
when it work and when it not work...
in my code i write remarks about not working point
import pymysql
try:
conn = pymysql.connect(host='localhost', user='root', password='1234', db='university')
conn.set_charset('utf8')
curs = conn.cursor(pymysql.cursors.DictCursor) #Dictionary cursor 생성
# curs = conn.cursor()
print("Connected to MySQL")
sql = "SELECT sno, midterm, final from db_score where midterm >= 20 and final >= 20 order by sno"
# sql = "select* from db_score"
curs.execute(sql)
#this point not work :(
except Exception as e:
print(str(e))
finally:
if conn:
curs.close()
conn.close()
print("MySql connection is closed")
and fetchall() didnt work :(\
import pandas as pd
import pymysql
xl_file = 'db_score.xlsx'
df = pd.read_excel(xl_file)
tp = list(df.itertuples(index=False, name=None))
# ('sno', 'attendance', 'homework', 'discussion', 'midterm', 'final', 'score', 'grade')
try:
conn = pymysql.connect(host='localhost', user='root', password='1234', db='university')
conn.set_charset('utf8')
#curs = conn.cursor(pymysql.cursors.DictCursor)
curs = conn.cursor()
print("Connected to MySQL")
sql = "INSERT INTO db_score VALUES (%s, %s, %s, %s, %s, %s, %s, %s)"
for i in range(0, len(df.index)):
# print('hi')
curs.execute(sql, tp[i])
#why work i dont know because other part is not working
# sql2 = "SELECT* from db_score"
# curs.execute(sql2)
# try execute, but not work
records = curs.fetchall()
for row in records:
print("why didn't work")
print(row)
#print not work :(
conn.commit()
except Exception as e:
print(str(e))
conn.rollback()
finally:
if conn:
curs.close()
conn.close()
print("MySql connection is closed")
please comment why work and why not work please...
thanks for watching
db connection is so hard:(
I'm new to python and trying to work out how to insert some JSON into MySQL database in different python code. It works fine when run separately, but not works when i try to connect 2nd to 1st script. How to links 2nd python in 1st python script so it can works together?
I have 1st code like bellow, this code serves to send the image to API and generate a json file data.json
import glob
import argparse
import requests
import json
import time
import os
import cv2
import numpy as np
from pprint import pprint
import json_mysql
def main():
result = []
regions = ['id']
time_to_wait = np.inf
time_counter = 0
while True:
files = glob.glob(os.path.join("./image_dir*/*.jpg"))
files.sort(key=os.path.getmtime)
for file in files:
if os.path.isfile(file):
with open(file, 'rb') as fp:
response = requests.post(
'https://API/',
data=dict(regions=regions),
files=dict(upload=fp),
headers={'Authorization': 'Token ' + 'XXXX'})
result.append(response.json())
with open('data.json', 'w') as outfile:
json.dump(result, outfile)
time.sleep(1)
pprint(response.json())
os.remove(file)
time.sleep(1)
time_counter += 1
if time_counter > time_to_wait: break
print("waiting for file... ")
if __name__ == '__main__':
main()
json_mysql.create_db()
it generate json file like this:
And 2nd code is for create and store 'data.json' to MySQL database:
from urllib.request import urlopen
import urllib
import json
import sys
import pymysql
def dbconnect():
try:
db = pymysql.connect(
host="localhost",
user="root",
passwd="YYYY",
)
except Exception as e:
sys.exit("Can't connect to Database")
return db
def create_db():
db_name="plate_recognizer"
table_name="vehicles"
db = dbconnect()
cursor = db.cursor()
cursor.execute("SET sql_notes = 0;")
cursor.execute("CREATE DATABASE IF NOT EXISTS {}".format(db_name))
cursor.execute("SET sql_notes = 0;")
cursor.execute(
"""CREATE TABLE IF NOT EXISTS {}.{}(time varchar(150),plate varchar(20),region varchar(150), score varchar(20), filename varchar(50), tipe varchar(10));""".format(db_name, table_name))
cursor.execute("SET sql_notes = 1;")
with open('data.json') as f:
data = json.load(f)
for i in data:
cursor.execute(
"""INSERT INTO {}.{}(time, plate, region, score, filename, tipe) VALUES(%s, %s, %s, %s, %s, %s)
ON DUPLICATE KEY UPDATE plate =%s """.format
(db_name, table_name),
(i['timestamp'], i['results'][0]['plate'].upper(), i['results'][0]['region']['code'], i['results'][0]['score'], i['filename'], i['results'][0]['vehicle']['type'], i['results'][0]['plate'].upper()))
db.commit()
db.close()
if __name__ == '__main__':
create_db()
Thanks in advance.
You should divide your code in several methods. Each method should be responsible to perform a particular task, independent of other methods. For example you can use an insert_data() method to just insert your data.
# creates your db
def create_db():
db_name="plate_recognizer"
table_name="vehicles"
db = dbconnect()
cursor = db.cursor()
cursor.execute("SET sql_notes = 0;")
cursor.execute("CREATE DATABASE IF NOT EXISTS {}".format(db_name))
cursor.execute("SET sql_notes = 0;")
cursor.execute(
"""CREATE TABLE IF NOT EXISTS {}.{}(time varchar(150),plate varchar(20),region varchar(150), score varchar(20), filename varchar(50), tipe varchar(10));""".format(db_name, table_name))
cursor.execute("SET sql_notes = 1;")
# pass data to be inserted
def insert_data(cursor, data):
for i in data:
cursor.execute(
"""INSERT INTO {}.{}(time, plate, region, score, filename, tipe) VALUES(%s, %s, %s, %s, %s, %s)
ON DUPLICATE KEY UPDATE plate =%s """.format
(db_name, table_name),
(i['timestamp'], i['results'][0]['plate'].upper(), i['results'][0]['region']['code'], i['results'][0]['score'], i['filename'], i['results'][0]['vehicle']['type'], i['results'][0]['plate'].upper()))
db.commit()
db.close()
def main():
db = dbconnect()
cursor = db.cursor()
# your logic
while True:
files = glob.glob(os.path.join("./image_dir*/*.jpg"))
files.sort(key=os.path.getmtime)
for file in files:
if os.path.isfile(file):
with open(file, 'rb') as fp:
response = requests.post(
'https://API/',
data=dict(regions=regions),
files=dict(upload=fp),
headers={'Authorization': 'Token ' + 'XXXX'})
result.append(response.json())
with open('data.json', 'w') as outfile:
json.dump(result, outfile)
time.sleep(1)
pprint(response.json())
# insert data in db
insert_data(cursor, response.json())
os.remove(file)
time.sleep(1)
time_counter += 1
if time_counter > time_to_wait: break
print("waiting for file... ")
I am generating dummy data in the database and then publishing to cloud.
As soon as it is published, I want to delete that entry from the database.
I understand that I need to send the 'mid' value in the publish method then call deleteFromDb() function. But mid is always 1, even though I return ret.mid=index. Index is the primary key retrieved from the database.
def on_publish(unused_client,unused_userdata,mid):
print('on_publish')
deleteFromDb(mid)
def main() :
TableSchema="""
create table if not exists heartbeat(
id integer primary key autoincrement,
Date_Time text,
Heartbeat text
);
"""
while True:
conn = sqlite3.connect(dbFile)
print("Connected to db")
conn.execute('pragma foreign_keys = on')
conn.commit()
curs = conn.cursor()
print "Writing to db..."
sqlite3.complete_statement(TableSchema)
curs.executescript(TableSchema)
conn.commit()
rectime=strftime("%Y-%m-%d %H:%M:%S", gmtime())
res="ON"
curs.execute("insert into heartbeat (Date_Time, Heartbeat)
values (?,?)",[rectime,res])
conn.commit()
print "Inserted Heartbeat Data into Database."
for row in curs.execute("select * from heartbeat"):
index=row[0]
continue
conn.commit()
encoded_row=''
encoded_row=json.dumps(row) #Dumped in the form of str
print encoded_row
client = mqtt.Client(client_id=_CLIENT_ID)
client.username_pw_set (username='unused', password=create_jwt(project_id, ssl_private_key_filepath, ssl_algorithm))
client=mqtt.Client()
client.on_connect = on_connect
client.on_publish=on_publish
client.on_message = on_message
client.tls_set(ca_certs=root_cert_filepath)
client.connect('mqtt.googleapis.com', 8883,60)
client.loop_start()
ret=client.publish(_MQTT_TOPIC,encoded_row,qos=1)
time.sleep(0.5)
ret.mid=index
client.loop_stop()
#print(ret.mid)
curs.close()
conn.close()
I am assuming you are using sqlite3 so you need to connect and execute a delete statement to delete and then commit() to make changes you saved. Try like this
import sqlite3
deleteFromDb(mid):
con=sqlite3.connect("mydatabase.db") #your db name
cursor=con.cursor()
cursor.execute("DELETE FROM TABLE_NAME WHERE ID = %s" % mid)
con.commit()
con.close()
I am using mysqldb/python to push some data into a mysql db.
The script parses a bunch of XML files for the data.
The MySQL server seems to quit and give me a '#2002 - The server is not responding (or the local MySQL server's socket is not correctly configured)' error midway through the transactions - in a different place every time I run it (so I am assuming its not a specific piece of data that is making it fall over...)
It works perfectly until it reaches ~12 or 13 file and it gives me this error:
Error 2003: Can't connect to MySQL server on 'localhost' (10055)
Traceback (most recent call last):
File "sigFileParser.py", line 113, in <module>
doParser(sigfile_filename)
File "sigFileParser.py", line 106, in
doParser
doFormatsPush(packedFormats)
File "sigFileParser.py", line 27, in
doFormatsPush
sys.exit (1)
NameError: global name 'sys' is not defined
Once the error occurs I can not get into MySQL console or via PHOPmyadmin
If I leave if for a while, I can get back into MySQL
MySQL tables:
CREATE TABLE IF NOT EXISTS patterns
(Version int(3),
DateCreated DATETIME,
SigID int(4),
SigSpecificity CHAR(10),
ByteSeqReference CHAR(12),
MinFragLength int(4),
Position int(4),
SubSeqMaxOffset int(4),
SubSeqMinOffset int(4),
Pattern TEXT)
and
CREATE TABLE IF NOT EXISTS formats
(Version int(3),
DateCreated DATETIME,
FormatID int(4),
FormatName TEXT,
PUID TEXT,
FormatVersion TEXT,
FormatMIMEType TEXT,
InternalSignatureID int(4),
Extension TEXT,
HasPriorityOverFileFormatID int(4))
Py code
from lxml import etree
import re, os, MySQLdb
def doPatternPush(packedPatterns):
try:
db = MySQLdb.connect (host = "localhost", user = "root", passwd = "", db = "sigfiles")
c = db.cursor()
c.execute('''INSERT INTO sigfiles.patterns
(Version,DateCreated,SigID,SigSpecificity,ByteSeqReference,MinFragLength,Position,SubSeqMaxOffset,SubSeqMinOffset,Pattern)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)''', packedPatterns)
db.commit()
except MySQLdb.Error, e:
print "Error %d: %s" % (e.args[0], e.args[1])
sys.exit (1)
return (db)
def doFormatsPush(packedFormats):
try:
db = MySQLdb.connect (host = "localhost", user = "root", passwd = "", db = "sigfiles")
c = db.cursor()
c.execute('''INSERT INTO sigfiles.formats
(Version,DateCreated,FormatID,FormatName,PUID,FormatVersion,FormatMIMEType,InternalSignatureID,Extension,HasPriorityOverFileFormatID)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)''', packedFormats)
db.commit()
except MySQLdb.Error, e:
print "Error %d: %s" % (e.args[0], e.args[1])
sys.exit (1)
return(db)
def doParser(sigfile_filename):
tree = etree.parse(sigfile_filename)
root = tree.getroot()
attributes = root.attrib
if 'DateCreated' in root.attrib:
DateCreated = (attributes["DateCreated"])
if 'Version' in root.attrib:
Version = (attributes["Version"])
##--------- get internal sig details ------------------
for a in range (len(root[0])): #loops for sig ID
attributes = root[0][a].attrib
SigID=(attributes["ID"])
SigSpecificity = (attributes["Specificity"])
for b in range (len(root[0][a])): # loops for sequence pattern inside each sig
attributes = root[0][a][b].attrib
if 'Reference' in root[0][a][b].attrib:
ByteSeqReference = (attributes["Reference"])
else:
ByteSeqReference = "NULL"
attributes = root[0][a][b][0].attrib
if 'MinFragLength' in root[0][a][b][0].attrib:
MinFragLength=(attributes["MinFragLength"])
else:
MinFragLength=''
if 'Position' in root[0][a].attrib:
Position=(attributes["Position"])
else:
Position=''
if 'SubSeqMaxOffset' in root[0][a][b][0].attrib:
SubSeqMaxOffset=(attributes["SubSeqMaxOffset"])
else:
SubSeqMaxOffsee = ''
if 'SubSeqMinOffset' in root[0][a][b][0].attrib:
SubSeqMinOffset=(attributes["SubSeqMinOffset"])
else:
SubSeqMinOffset = ''
Pattern = root[0][a][b][0][0].text
packedPatterns = [Version,DateCreated,SigID,SigSpecificity,ByteSeqReference,MinFragLength,Position,SubSeqMaxOffset,SubSeqMinOffset,Pattern]
doPatternPush(packedPatterns)
##-------- get format ID details-------------
for a in range (len(root[1])):
attributes = root[1][a].attrib
if 'ID' in root[1][a].attrib:
FormatID = (attributes['ID'])
else:
FormatID = "NULL"
if 'Name' in root[1][a].attrib:
FormatName = (attributes['Name'])
else:
FormatName = "NULL"
if 'PUID' in root[1][a].attrib:
PUID = (attributes['PUID'])
else:
PUID = "NULL"
if 'Version' in root[1][a].attrib:
FormatVersion = (attributes['Version'])
else:
FormatVersion = "NULL"
if 'MIMEType' in root[1][a].attrib:
FormatMIMEType = (attributes['MIMEType'])
else:
FormatMIMEType = "NULL"
InternalSignatureID,Extension,HasPriorityOverFileFormatID = ('', 'NULL', '')
for b in range (len(root[1][a])): #extracts the tags for each format ID
tagType = root[1][a][b].tag
tagText = root[1][a][b].text
tagType = re.sub('{http://www.nationalarchives.gov.uk/pronom/SignatureFile}', '', tagType)
if tagType == 'InternalSignatureID':
InternalSignatureID = tagText
elif tagType == 'Extension':
Extension = tagText
HasPriorityOverFileFormatID = ''
else:
HasPriorityOverFileFormatID = tagText
Extension = 'NULL'
packedFormats = [Version,DateCreated,FormatID,FormatName,PUID,FormatVersion,FormatMIMEType,InternalSignatureID,Extension,HasPriorityOverFileFormatID]
doFormatsPush(packedFormats)
if __name__ == "__main__":
path = "C:\Users\NDHA\Desktop\droid sigs all"
for (path, dirs, files) in os.walk(path):
for file in files:
sigfile_filename = str(path)+"\\"+str(file)
doParser(sigfile_filename)
print sigfile_filename
db.close()
All the XML comes from here: http://www.nationalarchives.gov.uk/aboutapps/pronom/droid-signature-files.htm
The error you get tells you exactly what's wrong
NameError: global name 'sys' is not defined
You don't import sys in your python file.
As for the db connection, if your socket is not placed in /tmp/mysql.sock, you can specify where to look for it when you try to connect to the db using the unix_socket parameter.
Try:
db = MySQLdb.connect (unix_socket = 'path_to_sock', host = "localhost",
user = "root", passwd = "", db = "sigfiles")
Where you replace 'path_to_sock' with the actual path to the mysql sock.
Other stuff you should check in case that isn't the issue:
Check to make sure the username/password combination is correct
Try stopping and re-starting the mysqld service
Check the error log files for more specific errors
This is your first error:
Error 2003: Can't connect to MySQL server on 'localhost' (10055)
It seems you disconnect from MySQL at some point. Check your code and see if you're explicitly or implicitly getting disconnected from the server and also check if your MySQL server is still listening to connections... maybe you're killing the server from outside your app... who knows? :)