Closed. This question does not meet Stack Overflow guidelines. It is not currently accepting answers.
This question appears to be off-topic because it lacks sufficient information to diagnose the problem. Describe your problem in more detail or include a minimal example in the question itself.
Closed 8 years ago.
Improve this question
I have two almost identical programs. One works, the other does not. The program that is not working displays the error "not enough arguments for format string". It probably has something to do with the "%" in the variable dbname, but I can't figure out why one program works and the other does not. In both programs I'm attempting to use a wildcard in a SELECT statement with LIKE.
Working program:
import subprocess
import sys
import commands
from sqlalchemy import create_engine
from sqlalchemy import Date, DateTime
from sqlalchemy import create_engine
from sqlalchemy import MetaData, Column, Table, ForeignKey
from sqlalchemy import Integer, String
from sqlalchemy.sql import select
engine = create_engine('mysql://UID:PASS#999.999.99.99:9999/access_benchmark_staging', echo=True)
dest = engine.connect()
dest.execute("truncate table TABAUTH")
def dbapull(applid, ssid, host, port, dbname):
print "dbapull " + dbname + ""
source = pyodbc.connect('Driver={IBM DB2 ODBC DRIVER};Database=' + ssid +';Hostname=' + host + ';Port=' + port + ';Protocol=TCPIP;Uid=user;Pwd=password', echo=True)
src = source.cursor()
src.execute("SELECT DISTINCT SUBSTR(CURRENT SERVER,1,7) AS SSID, SUBSTR(B.GRANTEE,1,8) AS GRANTEE, B.UPDATEAUTH AS U, B.INSERTAUTH AS I, B.DELETEAUTH AS D, A.CREATOR, B.DBNAME, B.TTNAME, B.ALTERAUTH AS C FROM " + ssid + ".SYSIBM.SYSTABLES A LEFT JOIN " + ssid + ".SYSIBM.SYSTABAUTH B ON A.CREATOR = B.TCREATOR AND A.NAME = B.TTNAME WHERE A.CREATOR IN ('PFPROD','PGPROD','PSPROD','PS','PROD') AND A.DBNAME LIKE " + dbname + " AND (B.UPDATEAUTH <> ' ' OR B.INSERTAUTH <> ' ' OR B.DELETEAUTH <> ' ') AND A.TYPE IN ('T','V') AND B.GRANTEETYPE = ' ' AND A.NAME NOT IN ('DSN_VIEWREF_TABLE','DSN_PGRANGE_TABLE','DSN_SORTKEY_TABLE','DSN_SORT_TABLE','DSN_DETCOST_TABLE','DSN_FILTER_TABLE','DSN_PTASK_TABLE','DSN_STATEMNT_TABLE','DSN_PGROUP_TABLE','DSN_STRUCT_TABLE','DSN_PREDICAT_TABLE','PLAN_TABLE') ORDER BY 2")
for row in src:
#print (row)
row[0] = str(row[0]).strip()
row[1] = str(row[1]).strip()
row[2] = str(row[2]).strip()
row[3] = str(row[3]).strip()
row[4] = str(row[4]).strip()
row[5] = str(row[5]).strip()
row[6] = str(row[6]).strip()
row[7] = str(row[7]).strip()
row[8] = str(row[8]).strip()
result = dest.execute("insert ignore into TABAUTH values ('" + applid + "','" + row[0] + "','" + row[1] + "','" + row[2] + "','" + row[3] + "','" + row[4] + "','" + row[5] + "','" + row[6] + "','" + row[7] + "','" + row[8] + "')")
dbapull("AAA", "BBB", "CCC", "DDD", "'%PMC%'")
dest.close()
Non-working program:
import subprocess
import sys
import commands
from sqlalchemy import create_engine
from sqlalchemy import Date, DateTime
from sqlalchemy import create_engine
from sqlalchemy import MetaData, Column, Table, ForeignKey
from sqlalchemy import Integer, String
from sqlalchemy.sql import select
engine = create_engine('mysql://UID:PASS#999.999.99.99:9999/access_benchmark_staging', echo=True)
dest = engine.connect()
dest.execute("truncate table DS_Users")
def userpull(appl, dbname):
print "DS User pull " + dbname + " "
source = create_engine('mysql://UID:PASS#999.999.99.99:9999/access_benchmark_staging', echo=True)
src = engine.connect()
src.execute("SELECT MF.profile_name AS profile_name, MF.groupuser_access as groupuser_access, MF.group_id as group_id, MF.user_id as user_id, MF.user_name as user_name, MF.default_group as default_group, MF.last_racinit as last_racinit, MF.password_last_changed_date as password_last_changed_date, MF.user_id_status as user_id_status, MF.creation_date as creation_date, ldap.uid as uid, ldap.company as company, ldap.emp_name as emp_name, ldap.title as title, ldap.contract_exp as contact_exp, ldap.dept_name as dept_name, ldap.emp_status as emp_status, ldap.disabled_date as disabled_date, ldap.term_date as term_date, ldap.bus_unit as bus_unit, ldap.manager_id as manager_id FROM (SELECT DST.profile_name, DST.groupuser_access, GRP.group_id, USR.user_id, USR.user_name, USR.default_group, USR.last_racinit, USR.password_last_changed_date, USR.user_id_status, USR.creation_date FROM AU_KRC_USER_REPORT USR INNER JOIN AU_KRC_GROUP_REPORT GRP ON USR.user_id = GRP.user_id INNER JOIN AU_KRC_DATASET_REPORT DST ON GRP.group_id = DST.groupuser_id WHERE (DST.profile_name LIKE " + dbname + " AND DST.profile_name NOT IN ('" + appl + ".SYSINFO.ABEND')) AND DST.groupuser_access IN ('UPDAT', 'ALTER') AND DST.groupuser_type = 'GROUP' ) MF LEFT OUTER JOIN ldap.ldap_raw ldap ON MF.user_id = ldap.kmart_mf GROUP BY MF.group_id, MF.user_id, MF.user_name, MF.default_group, MF.last_racinit, MF.password_last_changed_date, MF.user_id_status, MF.creation_date, ldap.uid, ldap.company, ldap.emp_name,ldap.title, ldap.contract_exp, ldap.dept_name, ldap.emp_status, ldap.disabled_date, ldap.term_date, ldap.bus_unit, ldap.manager_id")
for row in src:
#print (row)
row[0] = str(row[0]).strip()
row[1] = str(row[1]).strip()
row[2] = str(row[2]).strip()
row[3] = str(row[3]).strip()
row[4] = str(row[4]).strip()
row[5] = str(row[5]).strip()
row[6] = str(row[6]).strip()
row[7] = str(row[7]).strip()
row[8] = str(row[8]).strip()
row[9] = str(row[9]).strip()
row[10] = str(row[10]).strip()
row[11] = str(row[11]).strip()
row[12] = str(row[12]).strip()
row[13] = str(row[13]).strip()
row[14] = str(row[14]).strip()
row[15] = str(row[15]).strip()
row[16] = str(row[16]).strip()
row[17] = str(row[17]).strip()
row[18] = str(row[18]).strip()
row[19] = str(row[19]).strip()
row[20] = str(row[20]).strip()
result = dest.execute("insert ignore into DS_Users values ('" + appl +"','" + row[0] + "','" + row[1] + "','" + row[2] + "','" + row[3] + "','" + row[4] + "','" + row[5] + "','" + row[6] + "','" + row[7] + "','" + row[8] + "','" + row[9] + "','" + row[10] + "','" + row[11] + "','" + row[12] + "','" + row[13] + "','" + row[14] + "','" + row[15] + "','" + row[16] + "','" + row[17] + "','" + row[18] + "','" + row[19] + "','" + row[20] + "')")
userpull("PP", "'%PP.%'")
Try to put your sql in a transaction and check if the driver you are using for the DB support the "%" for queries.
Related
I am writing a script that transfers data from an Access database to a MySQL database. I'm am trying to generate a query similar to below:
INSERT into customers (firstname, lastname) value ('Charlie', "D'Amelio");
However, MySQL does not like double quotes like those listed above. I wrote a clunky function to try to replace the ' in D'Amelio with a '. Here is the whole function to create the SQL statement below:
def dictionary_output(dict):
output = "INSERT into lefm_customers "
fields = "(id, "
vl = "('" + id_gen() + "', "
for key in dict.keys():
# print(dict[key])
if str(dict[key]) == 'None' or str(dict[key]) == "":
pass
elif "'" in str(dict[key]):
fields = fields + str(key) + ", "
string = ""
for character in string:
if character == "'":
string += r"\'"
else:
string += character
vl = "'" + string + "', "
else:
fields = fields + str(key) + ", "
vl = vl + "'" + str(dict[key]) + "', "
fields = fields[:-2] + ")"
vl = vl[:-2] + ");"
return "INSERT into lefm_customers " + fields + " values " + vl
Currently it is just ignoring that value altogether. Any tips on what to replace ' with or how I can improve my function? Thank you!
This fixed it:
def dictionary_output(dict):
lst = []
output = "INSERT into lefm_customers "
fields = "(id, "
vl = "('" + id_gen() + "', "
for key in dict.keys():
# print(dict[key])
if str(dict[key]) == 'None' or str(dict[key]) == "":
pass
else:
fields = fields + str(key) + ", "
vl = vl + "%s, "
lst.append(dict[key])
fields = fields[:-2] + ")"
vl = vl[:-2] + ");"
return ("INSERT into lefm_customers " + fields + " values " + vl, lst)
for name in access_dict:
if str(name) not in mysql_dict.keys():
try:
statement = dictionary_output(access_dict[name])
mysql_cursor.execute(statement[0], statement[1])
print('attempting ' + str(name))
db_connection.commit()
print("Success!")
except:
print('something went wrong')
You can just call Python's replace. Example in the terminal:
>>> s = "D'Amelio"
>>> s.replace("'", "'")
"D'Amelio"
In this case the first argument is the single quote ' and the second is the acute accent '.
I'm trying to analyze a sqlite3 file and printing the results to a text file. If i test the code with print it all works fine. When i write it to a file it cuts out at the same point every time.
import sqlite3
import datetime
import time
conn = sqlite3.connect("History.sqlite")
curs = conn.cursor()
results = curs.execute("SELECT visits.id, visits.visit_time, urls.url, urls.visit_count \
FROM visits INNER JOIN urls ON urls.id = visits.url \
ORDER BY visits.id;")
exportfile = open('chrome_report.txt', 'w')
for row in results:
timestamp = row[1]
epoch_start = datetime.datetime(1601,1,1)
delta = datetime.timedelta(microseconds=int(timestamp))
fulltime = epoch_start + delta
string = str(fulltime)
timeprint = string[:19]
exportfile.write("ID: " + str(row[0]) + "\t")
exportfile.write("visit time: " + str(timeprint) + "\t")
exportfile.write("Url: " + str(row[2]) + "\t")
exportfile.write("Visit count: " + str(row[3]))
exportfile.write("\n")
print "ID: " + str(row[0]) + "\t"
print "visit time: " + str(timeprint) + "\t"
print "Url: " + str(row[2]) + "\t"
print "Visit count: " + str(row[3])
print "\n"
conn.close()
So the print results give the proper result but the export to the file stops in the middle of a url.
OK, I would start by replacing the for loop with the one below
with open('chrome_report.txt', 'w') as exportfile:
for row in results:
try:
timestamp = row[1]
epoch_start = datetime.datetime(1601,1,1)
delta = datetime.timedelta(microseconds=int(timestamp))
fulltime = epoch_start + delta
string = str(fulltime)
timeprint = string[:19]
exportfile.write("ID: " + str(row[0]) + "\t")
exportfile.write("visit time: " + str(timeprint) + "\t")
exportfile.write("Url: " + str(row[2]) + "\t")
exportfile.write("Visit count: " + str(row[3]))
exportfile.write("\n")
print "ID: " + str(row[0]) + "\t"
print "visit time: " + str(timeprint) + "\t"
print "Url: " + str(row[2]) + "\t"
print "Visit count: " + str(row[3])
print "\n"
except Exception as err:
print(err)
By using the "with" statement (context manager) we eliminate the need to close the file. By using the try/except we capture the error and print it. This will show you where your code is failing and why.
I have a python code, which do a SELECT from mysql database, then write result to files.
In result, files /tmp/ifcfg-* is ok, but in file /tmp/route i see only last line from database query, it look like:
10.130.48.0/23 via 10.130.48.7
but i expect
10.130.48.0/23 via 10.130.48.7
192.168.156.0/22 via 192.168.156.91
10.16.234.0/29 via 10.16.234.2
10.16.234.8/29 via 10.16.234.10
what is the reason ?
get_settings = "SELECT id_srv, \
vlan, \
phys_dev_srv, \
onboot, \
inet_ntoa(subnet_ipv4), \
prefix, \
inet_ntoa(int_ipv4), \
inet_ntoa(dns_srv_01), \
inet_ntoa(dns_srv_02), \
dns_domain \
FROM interfaces_ipv4 WHERE id_srv = '%i'" % (curr_srv_id);
cursor = conn.cursor()
cursor.execute(get_settings)
rows = cursor.fetchall()
for row in rows:
with open('/tmp/ifcfg-' + row[2] + '.' + str(row[1]), 'w+') as result, \
open('/tmp/route', 'w+') as route:
result.write('# vlan ' + str(row[1]) + '\n')
result.write('VLAN=yes' + '\n')
result.write('BOOTPROTO=static' + '\n')
result.write('NAME=' + row[2] + '.' + str(row[1]) + '\n')
result.write('DEVICE=' + row[2] + '.' + str(row[1]) + '\n')
result.write('PHYSDEV=' + row[2] + '\n')
result.write('ONBOOT=' + row[3] + '\n')
result.write('IPADDR=' + row[6] + '\n')
result.write('PREFIX=' + str(row[5]) + '\n')
result.write('DNS1=' + row[7] + '\n')
result.write('DNS2=' + row[8] + '\n')
result.write('DOMAIN=' + row[9] + '\n')
route.write(row[4] + '/' + str(row[5]) + ' via ' + row[6] +'\n')
Place the with statement outside the for-loop
Ex:
cursor = conn.cursor()
cursor.execute(get_settings)
rows = cursor.fetchall()
with open('/tmp/ifcfg-' + row[2] + '.' + str(row[1]), 'w+') as result, open('/tmp/route', 'w+') as route:
for row in rows:
result.write('# vlan ' + str(row[1]) + '\n')
result.write('VLAN=yes' + '\n')
result.write('BOOTPROTO=static' + '\n')
result.write('NAME=' + row[2] + '.' + str(row[1]) + '\n')
result.write('DEVICE=' + row[2] + '.' + str(row[1]) + '\n')
result.write('PHYSDEV=' + row[2] + '\n')
result.write('ONBOOT=' + row[3] + '\n')
result.write('IPADDR=' + row[6] + '\n')
result.write('PREFIX=' + str(row[5]) + '\n')
result.write('DNS1=' + row[7] + '\n')
result.write('DNS2=' + row[8] + '\n')
result.write('DOMAIN=' + row[9] + '\n')
route.write(row[4] + '/' + str(row[5]) + ' via ' + row[6] +'\n')
Your approach is overriding the content of the file because you are opening the file each time in the forloop and writing content again.
I am using pypyodbc to insert data into a database and when I use the cursor.execute() command I try to put the sql string and the parameters, but I get the following error:
SELECT uid FROM HP_DATA WHERE( hpName = ? AND processID = ? AND ipAddress = ? AND port = ? AND usernameTried = ? AND passwordTried = ? AND fileID = ?);
INSERT INTO HP_DATA_LOGIN_DETAIL(uid, attackDate, gmtOffset) VALUES(?, CONVERT(DATETIME, ?, 126), ?);
2016-04-19T05:40:58.000
('22007', '[22007] [Microsoft][ODBC SQL Server Driver][SQL Server]Conversion failed when converting date and/or time from character string.')
This is my code:
# Inserting the info of the file that is read into HP_DATA_LOG
# This is supposed to allow us to check in the future, what files are read/unread
print("Inserting File data into HP_DATA_LOG...")
log_file_date_read = datetime.datetime.today()
log_file_date_added = datetime.datetime.fromtimestamp(os.path.getctime(path)).strftime("%Y-%m-%d %H:%M:%S.%f")
file_size = os.path.getsize(path)
#log_sql = "INSERT INTO HP_DATA_LOG(dateRead, dateAdded, fileName, fileSize) VALUES("
#log_sql += "'" + str(log_file_date_read) + "', "
#log_sql += "'" + str(log_file_date_added) + "', "
#log_sql += "'" + path + "', "
#log_sql += "" + str(file_size) + ");"
log_params = (log_file_date_read, log_file_date_added, file_name, file_size)
log_sql = '''INSERT INTO HP_DATA_LOG(dateRead, dateAdded, fileName, fileSize) VALUES(?, ?, ?, ?);'''
print(log_sql)
cursor.execute(log_sql, log_params)
# Getting the auto-generated fileID from the table
print("Getting fileID...")
#get_fileID_sql = "SELECT fileID FROM HP_DATA_LOG WHERE "
#get_fileID_sql += "(dateRead = '" + str(log_file_date_read) + "'"
#get_fileID_sql += " AND dateAdded = '" + str(log_file_date_added) + "'"
#get_fileID_sql += " AND fileName = '" + path + "'"
#get_fileID_sql += " AND fileSize = '" + str(file_size) + "');"
fileID_params = (log_file_date_read, log_file_date_added, file_name, file_size)
get_fileID_sql = '''SELECT fileID FROM HP_DATA_LOG WHERE (dateRead = ? AND dateAdded = ? AND fileName = ? AND fileSize = ?);'''
print(get_fileID_sql)
cursor.execute(get_fileID_sql, fileID_params)
fileID = cursor.fetchone()
# Logging the attack by Inserting the HoneyPot data into HP_DATA
hp_name = re.findall('-\d\d:\d\d\s(.*)\ssshd', line)
pid = re.findall('\ssshd-22\[(\d+)\]', line)
ip_add = re.findall('\sIP:\s(\d+.\d+.\d+.\d+)\s', line)
port = re.findall('\s.\d+\sPass(.*)Log\s', line)
if port == "2222":
port = '2222'
else:
port = '22'
username = re.findall('\sUsername:\s(.*)\sPas', line)
password = re.findall('\sPassword:\s(.*)', line)
#sql = "INSERT INTO HP_DATA(hpName, processID, ipAddress, port, usernameTried, passwordTried, fileID) VALUES("
#sql += "'" + hp_name[0] + "', "
#sql += str(int(pid[0])) + ", "
#sql += "'" + ip_add[0] + "', "
#sql += str(port) + ", "
#sql += "'" + username[0] + "', "
#sql += "'" + password[0] + "', "
#sql += str(list(fileID)[0]) + ");"
sql_params = (hp_name[0], pid[0], ip_add[0], port, username[0], password[0], fileID[0])
sql = '''INSERT INTO HP_DATA(hpName, processID, ipAddress, port, usernameTried, passwordTried, fileID) VALUES(?, ?, ?, ?, ?, ?, ?);'''
print(sql)
cursor.execute(sql, sql_params)
#
#user_sql = r"SELECT uid FROM HP_DATA WHERE("
#user_sql += "hpName = '" + hp_name[0] + "' AND "
#user_sql += "processID = " + str(int(pid[0])) + " AND "
#user_sql += "ipAddress = '" + ip_add[0] + "' AND "
#user_sql += "port = " + str(port) + " AND "
#user_sql += r"usernameTried = '" + username[0] + "' AND "
#user_sql += r"passwordTried = '" + password[0] + "' AND "
#user_sql += "fileID = " + str(list(fileID)[0]) + ");"
user_sql_params = (hp_name[0], pid[0], ip_add[0], port, username[0], password[0], fileID[0])
user_sql = '''SELECT uid FROM HP_DATA WHERE( hpName = ? AND processID = ? AND ipAddress = ? AND port = ? AND usernameTried = ? AND passwordTried = ? AND fileID = ?);'''
print(user_sql)
cursor.execute(user_sql, user_sql_params)
uid = cursor.fetchone()
# Inserting date and time information in order to prevent duplicates
attack_date = re.findall('(\d{4}-\d\d-\d\d)T', line)
timestamp = re.findall('T(\d\d:\d\d:\d\d.*).*-.*sshd', line)
attack_datetime = attack_date[0] + "T" + timestamp[0] + ".000"
gmt_offset = re.findall('\d\d:\d\d:\d\d.*-(\d\d:\d\d)\s', line)
#hp_detail_sql = r"INSERT INTO HP_DATA_LOGIN_DETAIL(uid, attackDate, attackTime, gmtOffset) VALUES("
#hp_detail_sql += "" + str(uid[0]) + ", "
#hp_detail_sql += "'" + attackDate[0] + "', "
#hp_detail_sql += "'" + timestamp[0] + "', "
#hp_detail_sql += "'" + gmt_offset[0] + "');"
hp_detail_sql_params = (uid[0], attack_datetime[0], gmt_offset[0])
hp_detail_sql = '''INSERT INTO HP_DATA_LOGIN_DETAIL(uid, attackDate, gmtOffset) VALUES(?, ?, ?);'''
print(hp_detail_sql)
print(attack_datetime)
cursor.execute(hp_detail_sql, hp_detail_sql_params)
print("Executed insert statements")
Use datetime.strptime() to convert the attack_datetime value to a datetime object before passing the value to SQL Server.
For example, passing a datetime formatted string fails with the same error message you receive
...
# assumes connection and cursor objects initialized
create_date_str = "2016-06-16T01:23:45.67890"
sql = "select name, create_date from sys.databases where create_date = ?"
rows = cursor.execute(sql, create_date_str).fetchall()
Raises
Traceback (most recent call last): File "", line 1, in
pyodbc.DataError: ('22007', '[22007] [Microsoft][SQL Server
Native Client 11.0][SQL Server]Conversion failed when converting date
and/or time from character string. (241) (SQLExecDirectW)')
While converting the datetime string to a datetime object succeeds
...
# convert datetime string to object, specifying input format
create_date = datetime.datetime.strptime(create_date_str, '%Y-%m-%dT%H:%M:%S.%f')
rows = cursor.execute(sql, create_date).fetchall()
I am using pyhton 2.7 to compare datasets. I call a select statement to pull from the two tables. str(row[4]) is always blank. I need to always write a "T" into that column.
Is there a way to write into str(row[4]) during the g.write statement I use in the below code?
This is part of my code:
## This is used to connect to our database and run the sql statement.
def RunSQLStatement(sql):
conn = psycopg2.connect(Connect2DB())
cursor = conn.cursor()
cursor.execute(sql)
alias_data = cursor.fetchall()
cursor.close()
conn.close()
return alias_data
def selectFoisp():
with open(outTemp_file, 'w') as g:
strGetDiffer = """SELECT c.foi_id, c.title, c.feat_type, c.feat_subtype, c.upd_type, p.foi_id, p.title
FROM foisp c, foisp_prior p
WHERE (c.foi_id = p.foi_id)
AND (c.title <> p.title)"""
extract_rows = RunSQLStatement(strGetDiffer)
for row in extract_rows:
g.write(str(row[0]) + ',' + str(row[1]) + ',' + str(row[2]) + ',' + str(row[3]) + ',' + str(row[4]) + ',' + str(row[5]) + ',' + str(row[6]) + '\n')
g.write(str(row[0]) + ',' + str(row[1]) + ',' + str(row[2]) + ',' + str(row[3]) + ', T,' + str(row[5]) + ',' + str(row[6]) + '\n')
or with format method:
g.write("{0}, {1}, {2}, {3}, {4}, {5}, {6}\n".format(row[0], row[1], row[2], row[3], "T", row[5], row[6])
If it is always blank, why don't you just replace str(row[4]) with 'T'? (as Michael has done)
That said you should be able to directly manipulate row[4] in your loop:
for row in extract_rows:
row[4] = 'T'
g.write(str(row[0]) + ',' + str(row[1]) + ',' + str(row[2]) + ',' + str(row[3]) + ',' + str(row[4]) + ',' + str(row[5]) + ',' + str(row[6]) + '\n')
Furthermore you should look into using the CSV module:
import csv
writer = csv.writer(g)
for row in extract_rows:
row[4] = 'T'
writer.writerow(row)
This will automatically sort out an iterable into the format you have specified (comma separated, although you can change some parameters if you want it to be separated with something else.) so you don't have to do lengthy string concatenations. It looks much cleaner that way as well.