I am writing a script that transfers data from an Access database to a MySQL database. I'm am trying to generate a query similar to below:
INSERT into customers (firstname, lastname) value ('Charlie', "D'Amelio");
However, MySQL does not like double quotes like those listed above. I wrote a clunky function to try to replace the ' in D'Amelio with a '. Here is the whole function to create the SQL statement below:
def dictionary_output(dict):
output = "INSERT into lefm_customers "
fields = "(id, "
vl = "('" + id_gen() + "', "
for key in dict.keys():
# print(dict[key])
if str(dict[key]) == 'None' or str(dict[key]) == "":
pass
elif "'" in str(dict[key]):
fields = fields + str(key) + ", "
string = ""
for character in string:
if character == "'":
string += r"\'"
else:
string += character
vl = "'" + string + "', "
else:
fields = fields + str(key) + ", "
vl = vl + "'" + str(dict[key]) + "', "
fields = fields[:-2] + ")"
vl = vl[:-2] + ");"
return "INSERT into lefm_customers " + fields + " values " + vl
Currently it is just ignoring that value altogether. Any tips on what to replace ' with or how I can improve my function? Thank you!
This fixed it:
def dictionary_output(dict):
lst = []
output = "INSERT into lefm_customers "
fields = "(id, "
vl = "('" + id_gen() + "', "
for key in dict.keys():
# print(dict[key])
if str(dict[key]) == 'None' or str(dict[key]) == "":
pass
else:
fields = fields + str(key) + ", "
vl = vl + "%s, "
lst.append(dict[key])
fields = fields[:-2] + ")"
vl = vl[:-2] + ");"
return ("INSERT into lefm_customers " + fields + " values " + vl, lst)
for name in access_dict:
if str(name) not in mysql_dict.keys():
try:
statement = dictionary_output(access_dict[name])
mysql_cursor.execute(statement[0], statement[1])
print('attempting ' + str(name))
db_connection.commit()
print("Success!")
except:
print('something went wrong')
You can just call Python's replace. Example in the terminal:
>>> s = "D'Amelio"
>>> s.replace("'", "'")
"D'Amelio"
In this case the first argument is the single quote ' and the second is the acute accent '.
Related
I am executing PostgreSQL13 queries coding them in python 3.9 using the psycopg2 library. I also am working with PostGIS extension over PostgreSQL.
Kindly look for the comment which points out the line which causes the syntax error. I am having trouble both understanding what is the syntax error and how to debug it since I need to execute PostgreSQL queries using python so any tips will be greatly appreciated.
def corefunc(rf, openConnection):
pcur = openConnection.cursor(name="pcur" + rf)
rcur = openConnection.cursor(name="rcur" + rf)
acur = openConnection.cursor()
rcur.execute("SELECT geom FROM " + rf)
for number in range (1, 5):
acur.execute("DROP TABLE IF EXISTS " + "pf"+ rf)
acur.execute("CREATE TABLE " + "pf" + rf + " (index integer, sums integer)")
pcur.execute("SELECT geom FROM " + "pf" + str(number))
row = 1
for each in rcur.fetchall():
if number == 1: acur.execute("INSERT INTO " + "pf" + rf + " (index, sums) VALUES (" + str(row) + ",0)")
for eachone in pcur.fetchall():
#-------------------------------------------- the statement below gives the syntax error
acur.execute("UPDATE TABLE " + "pf" + rf + " SET sums = sums + "\
+"ST_Contains(" + " ' " + each[0] + " ' " + ", " + " ' " + eachone[0] + " ' " + ")::int WHERE index = " + str(row))
row = row + 1
def parallelJoin (pointsTable, rectsTable, outputTable, outputPath, openConnection):
#Implement ParallelJoin Here.
cursor = openConnection.cursor()
cursor.execute("SELECT COUNT(*) FROM " + pointsTable)
size_data = (cursor.fetchall())[0][0]
for number in range(1, 5):
cursor.execute("DROP TABLE IF EXISTS pf" + str(number))
cursor.execute("CREATE TABLE pf" + str(number) + " AS SELECT * FROM "
+ pointsTable + " LIMIT " + str(size_data/4)
+ " OFFSET " + str(((number-1)*size_data)/4))
cursor.execute("SELECT COUNT(*) FROM " + rectsTable)
size_rects = (cursor.fetchall())[0][0]
for number in range(1, 5):
cursor.execute("DROP TABLE IF EXISTS rf" + str(number))
cursor.execute("CREATE TABLE rf" + str(number) + " AS SELECT * FROM "
+ pointsTable + " LIMIT " + str(size_rects/4)
+ " OFFSET " + str(((number - 1) * size_rects)/4))
threads = dict()
for number in range(0, 4):
threads[number] = threading.Thread(target=corefunc, args=("rf" + str(number + 1), openConnection))
threads[number].start()
break
while threads[0].is_alive() or threads[1].is_alive()\
or threads[2].is_alive() or threads[3].is_alive(): pass
# more shit to do
Nevermind, I just thought checking out the syntax from somewhere and I found the problem. To update table, the query begins with "UPDATE...." not "UPDATE TABLE.....".
This is the following code
pythonlist = ['Name','Mno']
datalist = ["qwerty",'234']
sql = "SELECT " + ",".join(pythonlist) + " FROM data WHERE name = '"+ "','".join(datalist) + "' INTO OUTFILE filename"
print(sql)
OUTPUT:
SELECT Name,Mno FROM data WHERE Name= 'qwerty','234'
DESIRED OUTPUT:
SELECT Name,Mno FROM data WHERE Name = 'qwerty' and Mno = 234
Do note the removal of quotations marks in 'mno'.
The reason I am doing this is due because the column names, as well as values corresponding it to, will change frequently
Code :
queryparams = {'Name': 'qwerty', 'Mno': '234'}
and_clause = []
[and_clause.append(' %s = %s ') for k,v in queryparams.items()]
and_clause_str = ' and '.join(and_clause)
sql = 'SELECT %s FROM data WHERE ' + and_clause_str
params = [','.join(queryparams.keys())]
for k,v in queryparams.items():
params.append(str(k))
params.append(str(v))
print(sql)
print(params)
cursor.execute(sql, params=tuple(params))
This works if you add 10/20 more items to dictionary .
Aswell as prevents SQL-injection : Using params to pass values instead of string-concatenation .
Try this:
data = {'Name': 'qwerty' , 'Mno' : '234'}
sql = "SELECT " + ", ".join(data.keys()) + " FROM data WHERE " + str(list(data.keys())[0]) + " = '" + \
str(data[list(data.keys())[0]]) + "' and " +\
str(list(data.keys())[1]) + " = " + str(data[list(data.keys())[1]])
print(sql)
I have to connect the sql database to python so that I can add new user data via python.
I have tried the int conversion which puts me in further trouble of null types dataset.
i have tried the bracket placement. It doesn't work.
import os
import datetime
import pyodbc
import sqlite3
file_open = open("filenames.txt","r")
path = 'C:\\Users\\Timble\\Desktop\\Face_recognition\\user-id_filenames\\'
flag_loc = 1
flag_proc = 0
flag_vis = 0
file_read_lines = file_open.readlines()
for line in file_read_lines:
for character in line:
if character == "_":
details = line.split("_")
now = datetime.datetime.now()
name = line
print("name:", name) #col-3
print("type of name:", type(name))
user_id = int(details[1])
print("user_id:", details[1]) #col-2
print("type of user_id:", type(user_id))
date = details[2]
print("date on which photo is taken:", details[2]) #col-4
print("type of data:",type(details[2]))
now = now.strftime("%Y-%m-%d %H:%M:%S")
print("Current date and time: ", now) #col-6
print("type of current date:", type(now))
path2 = path + details[1]
if os.path.exists(path2):
print(path2)
else:
os.makedirs(path2)
#break
date = str(date)
print("type of date", type(date))
user_id = str(user_id)
print("type of user_id", type(user_id))
name = str(name)
print("type of name",type(name))
now = str(now)
print("type of now", type(now))
flag_loc = str(flag_loc)
print("type loc flag", type(flag_loc))
flag_proc = str(flag_proc)
print("type proc flag", type(flag_proc))
flag_vis = str(flag_vis)
print("type vis flag", type(flag_vis))
conn = pyodbc.connect(
"DRIVER={SQl Server};"
"server=DESKTOP-3ORBD3I\MSSQL;"
"database=TimbleSecuritySystem;"
"uid=sa;"
"pwd=P#ssword")
cur = conn.cursor()
sqlInsertUser = "Insert Into retraining (date, user_id, image_name,location_flagged, processing_flagged, insert_date, visible)Values( "+ date + " , " + user_id + " , " + name + " , " + flag_loc + " , " + flag_proc + " , " + now + " , " + flag_vis + " )"
print(sqlInsertUser)
cur.execute(sqlInsertUser)
conn.commit()
break
file_open.close()
The actual results tell me that print(sqlInsertUser) prints all the right values.
I am expecting the execute command to work and sql data added there.
This line is the problem:
sqlInsertUser = "Insert Into retraining (date, user_id, image_name,location_flagged, processing_flagged, insert_date, visible)Values( "+ date + " , " + user_id + " , " + name + " , " + flag_loc + " , " + flag_proc + " , " + now + " , " + flag_vis + " )"
For example if name contains some invalid characters e.g. "[" or "]", then the execute call fails because the name string is not properly enclosed. (It should be enclosed in a pair of quote)
You can use the parameter substitution support in pyodbc e.g.
sqlInsertUser = "Insert Into retraining (date, user_id,
image_name, location_flagged, processing_flagged, insert_date,
visible) Values (?,?,?,?,?,?,?)"
then run
cur.execute(sqlInsertUser, date, user_id, name, flag_loc, flag_proc, now, flag_vis)
(My sample code above is untested. You might need to fix some syntax errors)
For more details about the syntax see https://www.python.org/dev/peps/pep-0249/#paramstyle or https://github.com/mkleehammer/pyodbc/wiki/Cursor
I'm trying to analyze a sqlite3 file and printing the results to a text file. If i test the code with print it all works fine. When i write it to a file it cuts out at the same point every time.
import sqlite3
import datetime
import time
conn = sqlite3.connect("History.sqlite")
curs = conn.cursor()
results = curs.execute("SELECT visits.id, visits.visit_time, urls.url, urls.visit_count \
FROM visits INNER JOIN urls ON urls.id = visits.url \
ORDER BY visits.id;")
exportfile = open('chrome_report.txt', 'w')
for row in results:
timestamp = row[1]
epoch_start = datetime.datetime(1601,1,1)
delta = datetime.timedelta(microseconds=int(timestamp))
fulltime = epoch_start + delta
string = str(fulltime)
timeprint = string[:19]
exportfile.write("ID: " + str(row[0]) + "\t")
exportfile.write("visit time: " + str(timeprint) + "\t")
exportfile.write("Url: " + str(row[2]) + "\t")
exportfile.write("Visit count: " + str(row[3]))
exportfile.write("\n")
print "ID: " + str(row[0]) + "\t"
print "visit time: " + str(timeprint) + "\t"
print "Url: " + str(row[2]) + "\t"
print "Visit count: " + str(row[3])
print "\n"
conn.close()
So the print results give the proper result but the export to the file stops in the middle of a url.
OK, I would start by replacing the for loop with the one below
with open('chrome_report.txt', 'w') as exportfile:
for row in results:
try:
timestamp = row[1]
epoch_start = datetime.datetime(1601,1,1)
delta = datetime.timedelta(microseconds=int(timestamp))
fulltime = epoch_start + delta
string = str(fulltime)
timeprint = string[:19]
exportfile.write("ID: " + str(row[0]) + "\t")
exportfile.write("visit time: " + str(timeprint) + "\t")
exportfile.write("Url: " + str(row[2]) + "\t")
exportfile.write("Visit count: " + str(row[3]))
exportfile.write("\n")
print "ID: " + str(row[0]) + "\t"
print "visit time: " + str(timeprint) + "\t"
print "Url: " + str(row[2]) + "\t"
print "Visit count: " + str(row[3])
print "\n"
except Exception as err:
print(err)
By using the "with" statement (context manager) we eliminate the need to close the file. By using the try/except we capture the error and print it. This will show you where your code is failing and why.
I am using pypyodbc to insert data into a database and when I use the cursor.execute() command I try to put the sql string and the parameters, but I get the following error:
SELECT uid FROM HP_DATA WHERE( hpName = ? AND processID = ? AND ipAddress = ? AND port = ? AND usernameTried = ? AND passwordTried = ? AND fileID = ?);
INSERT INTO HP_DATA_LOGIN_DETAIL(uid, attackDate, gmtOffset) VALUES(?, CONVERT(DATETIME, ?, 126), ?);
2016-04-19T05:40:58.000
('22007', '[22007] [Microsoft][ODBC SQL Server Driver][SQL Server]Conversion failed when converting date and/or time from character string.')
This is my code:
# Inserting the info of the file that is read into HP_DATA_LOG
# This is supposed to allow us to check in the future, what files are read/unread
print("Inserting File data into HP_DATA_LOG...")
log_file_date_read = datetime.datetime.today()
log_file_date_added = datetime.datetime.fromtimestamp(os.path.getctime(path)).strftime("%Y-%m-%d %H:%M:%S.%f")
file_size = os.path.getsize(path)
#log_sql = "INSERT INTO HP_DATA_LOG(dateRead, dateAdded, fileName, fileSize) VALUES("
#log_sql += "'" + str(log_file_date_read) + "', "
#log_sql += "'" + str(log_file_date_added) + "', "
#log_sql += "'" + path + "', "
#log_sql += "" + str(file_size) + ");"
log_params = (log_file_date_read, log_file_date_added, file_name, file_size)
log_sql = '''INSERT INTO HP_DATA_LOG(dateRead, dateAdded, fileName, fileSize) VALUES(?, ?, ?, ?);'''
print(log_sql)
cursor.execute(log_sql, log_params)
# Getting the auto-generated fileID from the table
print("Getting fileID...")
#get_fileID_sql = "SELECT fileID FROM HP_DATA_LOG WHERE "
#get_fileID_sql += "(dateRead = '" + str(log_file_date_read) + "'"
#get_fileID_sql += " AND dateAdded = '" + str(log_file_date_added) + "'"
#get_fileID_sql += " AND fileName = '" + path + "'"
#get_fileID_sql += " AND fileSize = '" + str(file_size) + "');"
fileID_params = (log_file_date_read, log_file_date_added, file_name, file_size)
get_fileID_sql = '''SELECT fileID FROM HP_DATA_LOG WHERE (dateRead = ? AND dateAdded = ? AND fileName = ? AND fileSize = ?);'''
print(get_fileID_sql)
cursor.execute(get_fileID_sql, fileID_params)
fileID = cursor.fetchone()
# Logging the attack by Inserting the HoneyPot data into HP_DATA
hp_name = re.findall('-\d\d:\d\d\s(.*)\ssshd', line)
pid = re.findall('\ssshd-22\[(\d+)\]', line)
ip_add = re.findall('\sIP:\s(\d+.\d+.\d+.\d+)\s', line)
port = re.findall('\s.\d+\sPass(.*)Log\s', line)
if port == "2222":
port = '2222'
else:
port = '22'
username = re.findall('\sUsername:\s(.*)\sPas', line)
password = re.findall('\sPassword:\s(.*)', line)
#sql = "INSERT INTO HP_DATA(hpName, processID, ipAddress, port, usernameTried, passwordTried, fileID) VALUES("
#sql += "'" + hp_name[0] + "', "
#sql += str(int(pid[0])) + ", "
#sql += "'" + ip_add[0] + "', "
#sql += str(port) + ", "
#sql += "'" + username[0] + "', "
#sql += "'" + password[0] + "', "
#sql += str(list(fileID)[0]) + ");"
sql_params = (hp_name[0], pid[0], ip_add[0], port, username[0], password[0], fileID[0])
sql = '''INSERT INTO HP_DATA(hpName, processID, ipAddress, port, usernameTried, passwordTried, fileID) VALUES(?, ?, ?, ?, ?, ?, ?);'''
print(sql)
cursor.execute(sql, sql_params)
#
#user_sql = r"SELECT uid FROM HP_DATA WHERE("
#user_sql += "hpName = '" + hp_name[0] + "' AND "
#user_sql += "processID = " + str(int(pid[0])) + " AND "
#user_sql += "ipAddress = '" + ip_add[0] + "' AND "
#user_sql += "port = " + str(port) + " AND "
#user_sql += r"usernameTried = '" + username[0] + "' AND "
#user_sql += r"passwordTried = '" + password[0] + "' AND "
#user_sql += "fileID = " + str(list(fileID)[0]) + ");"
user_sql_params = (hp_name[0], pid[0], ip_add[0], port, username[0], password[0], fileID[0])
user_sql = '''SELECT uid FROM HP_DATA WHERE( hpName = ? AND processID = ? AND ipAddress = ? AND port = ? AND usernameTried = ? AND passwordTried = ? AND fileID = ?);'''
print(user_sql)
cursor.execute(user_sql, user_sql_params)
uid = cursor.fetchone()
# Inserting date and time information in order to prevent duplicates
attack_date = re.findall('(\d{4}-\d\d-\d\d)T', line)
timestamp = re.findall('T(\d\d:\d\d:\d\d.*).*-.*sshd', line)
attack_datetime = attack_date[0] + "T" + timestamp[0] + ".000"
gmt_offset = re.findall('\d\d:\d\d:\d\d.*-(\d\d:\d\d)\s', line)
#hp_detail_sql = r"INSERT INTO HP_DATA_LOGIN_DETAIL(uid, attackDate, attackTime, gmtOffset) VALUES("
#hp_detail_sql += "" + str(uid[0]) + ", "
#hp_detail_sql += "'" + attackDate[0] + "', "
#hp_detail_sql += "'" + timestamp[0] + "', "
#hp_detail_sql += "'" + gmt_offset[0] + "');"
hp_detail_sql_params = (uid[0], attack_datetime[0], gmt_offset[0])
hp_detail_sql = '''INSERT INTO HP_DATA_LOGIN_DETAIL(uid, attackDate, gmtOffset) VALUES(?, ?, ?);'''
print(hp_detail_sql)
print(attack_datetime)
cursor.execute(hp_detail_sql, hp_detail_sql_params)
print("Executed insert statements")
Use datetime.strptime() to convert the attack_datetime value to a datetime object before passing the value to SQL Server.
For example, passing a datetime formatted string fails with the same error message you receive
...
# assumes connection and cursor objects initialized
create_date_str = "2016-06-16T01:23:45.67890"
sql = "select name, create_date from sys.databases where create_date = ?"
rows = cursor.execute(sql, create_date_str).fetchall()
Raises
Traceback (most recent call last): File "", line 1, in
pyodbc.DataError: ('22007', '[22007] [Microsoft][SQL Server
Native Client 11.0][SQL Server]Conversion failed when converting date
and/or time from character string. (241) (SQLExecDirectW)')
While converting the datetime string to a datetime object succeeds
...
# convert datetime string to object, specifying input format
create_date = datetime.datetime.strptime(create_date_str, '%Y-%m-%dT%H:%M:%S.%f')
rows = cursor.execute(sql, create_date).fetchall()