I am trying to generate a csv file for each query output. I have multiple select queries (queries.sql) in a single SQL file and i am looping through it to execute in database and write each query output to its own csv file. When i execute the code all queries are executing in the database but only the last query result set is being written to csv file, rest all csv files are with no records. Any help is appreciated.
col_pattern = "(^|[_-])SSN#?($|[_-])|^SS#?$|(^|[_-])(SSN|SOC.*SEC.*).?(ID|NO|NUMBERS?|NUM|NBR|#)($|[_-])|^SOCIAL.?SEC(URITY)?#?$"
SQL = "select OWNER||'_'||TABLE_NAME||'_'||column_name from ALL_TAB_COLS where REGEXP_LIKE (column_name, :1) and owner NOT IN ('SYS','SYSMAN') order by table_name,column_name"
cursor.execute(SQL,(col_pattern,))
for row_data in cursor:
if not row_data[0].startswith('BIN$'):
fileName = row_data[0]
csv_file_dest = "/u01/exp/test/identity/csvdata/"+ fileName + ".csv"
outputFile = open(csv_file_dest,'w') # 'wb'
output = csv.writer(outputFile, dialect='excel')
f = open('/u01/exp/test/identity/queries.sql')
full_sql = f.read()
sql_commands = full_sql.replace('\n', "").split(';')[:-1]
#print(sql_commands)
for sql_command in sql_commands:
curs2 = cursor.execute(sql_command)
if printHeader: # add column headers if requested
cols = []
for col in curs2.description:
cols.append(col[0])
output.writerow(cols)
for row_data in curs2: # add table rows
output.writerow(row_data)
outputFile.close()
Looks like it is because all the SQL data is being written to whatever the last thing is that the variable "output" was set to. So the file that the variable "output" is set to is just being constantly overwritten.
Related
I'm using python to import from Excel information to my sqlite3 database. I need to avoid the same record to be added twice, but I'd also need to know which are the records added succeffully and which are the one that were already presented in the database. This is how I'm importing with pandas:
def import(self):
file = filedialog.askopenfilename(filetypes=(("Excel File", "*.csv"), ("All Files","*.*")), title="Select Excel")
with open(file, "r", encoding='utf-8') as f:
conn = sqlite3.connect('database/save.db')
double = []
ok = []
c = conn.cursor()
c.execute("SELECT rowid, * FROM record")
r = c.fetchall()
imported = pd.read_csv(f, keep_default_na=False, sep=';')
imported.to_sql('record', conn, if_exists='replace', index = False)
for row in r:
if row in imported:
double.append(row)
else:
ok.append(row)
conn.commit()
conn.close()
Using the for loop I'm able to save the duplicate row to the list but not the one inserted correctly which is always empty
Thanks for the help!
I am trying to insert data by chunks from a CSV files in the folder, but I cannot get the SQLITE insert query right. I was able to perform it without the lists, so I know that the data is correct.
However when I use the lists I get the error: sqlite3.ProgrammingError: Incorrect number of bindings supplied. The current statement uses 10, and there are 65 supplied.
Any ideas?
import csv, sqlite3, time, os
def chunks(data, rows=10000):
data = list(data)
for i in range(0, len(data), rows):
yield data[i:i+rows]
if __name__ == "__main__":
datab = 'MYDB'
con=sqlite3.connect(datab+'.db')
con.text_factory = str
cur = con.cursor()
koko = 'C:\\MYFOLDER\\'
print(koko)
directory = koko
print(directory)
for file in os.listdir(directory):
for searchfile, csvfile, csvcolumn, tablecolumn, table, valuemark, valcsvcolumn in zip(['USR02_FINAL.csv'],
['USR02_FINAL.csv'],
[['SYS,MANDT, BNAME, GLTGV, GLTGB, USTYP, CLASS, UFLAG, ERDAT, TRDAT']],
[['SYS,MANDT, BNAME, GLTGV2, GLTGB2, USTYP, CLASS, UFLAG, ERDAT2, TRDAT2']],
['USR_02_ALL_RAW2'],
[['?,?,?,?,?,?,?,?,?,?']],
[['SYS,MANDT, BNAME, GLTGV, GLTGB, USTYP, CLASS, UFLAG, ERDAT, TRDAT']]):
if file.endswith(searchfile):
fileinsert = directory + '\\' + csvfile
csvData = csv.reader(open(fileinsert, "rt"))
divData = chunks(csvData) # divide into 10000 rows each
for chunk in divData:
cur.execute('BEGIN TRANSACTION')
for csvcolumn in chunk:
print(searchfile, csvfile, csvcolumn, tablecolumn, table, valuemark, valcsvcolumn)
cur.execute("""INSERT OR IGNORE INTO """ + table +""" ("""+ ', '.join(tablecolumn) +""") VALUES ("""+ ', '.join(valuemark)+""")""",( ', '.join(valcsvcolumn)))
cur.execute('COMMIT')
Look at the loops:
for chunk in divData:
# ...
for csvcolumn in chunk:
# ...
...join(valcsvcolumn)
I see that you only use csvcolumn in the print, but not in the insert statement; it's using valcsvcolumn which is an unrelated thing. Probably this is the problem.
I am working with a text file (ClassTest.txt) and pandas. The text file has 3, tab-separated columns: Title, Description, and Category - Title and Description are normal strings and Category is a (non-zero) integer.
I was gathering the data as follows:
data = pd.read_table('ClassTest.txt')
feature_names = ['Title', 'Description']
X = data[feature_names]
y = data['Category']
However, because values in the Description column can themselves contain new lines, the 'y' DataFrame contains too many rows because of most of the items in the Description column having multiple lines. I attempted to get around this by making the newline character in the file to be '|' (by repopulating it) and using:
data = pd.read_table('ClassTest.txt', lineterminator='|')
X = data[feature_names]
y = data['Category']
This time, I get the error:
pandas.errors.ParserError: Error tokenizing data. C error: Expected 3 fields in line 20, saw 5
Can anyone help me with this issue?
EDIT: Adding previous code
con = lite.connect('JobDetails.db')
cur = con.cursor()
cur.execute('''SELECT Title, Description, Category FROM ReviewJobs''')
results = [list(each) for each in cur.fetchall()]
cur.execute('''SELECT Title, Description, Category FROM Jobs''')
for each in cur.fetchall():
results.append(list(each))
a = open('ClassTest.txt', 'ab')
newLine = "|"
a.write(u''.join(c for c in 'Title\tDescription\tCategory' + newLine).encode('utf-8'))
for r in results:
toWrite = "".encode('utf-8')
title = u''.join(c for c in r[0].replace("\n", " ")).encode('utf-8') + "\t".encode('utf-8')
description = u''.join(c for c in r[1]).encode('utf-8') + "\t".encode('utf-8')
toWrite += title + description
toWrite += str(r[2]).encode('utf-8') + newLine.encode('utf-8')
a.write(toWrite)
a.close()
pandas.read_table() is deprecated – use read_csv() instead. And then really use the CSV format instead of writing lots of code to write something similar that can't cope with record or field delimiters within fields. There's the csv module in the Python standard library.
Opening the file as text file and passing the encoding to open() spares you from encoding everything yourself in different places.
#!/usr/bin/env python3
from contextlib import closing
import csv
import sqlite3
def main():
with sqlite3.connect("JobDetails.db") as connection:
with closing(connection.cursor()) as cursor:
#
# TODO Having two tables with the same columns for essentially
# the same kind of records smells like a broken DB design.
#
rows = list()
for table_name in ["reviewjobs", "jobs"]:
cursor.execute(
f"SELECT title, description, category FROM {table_name}"
)
rows.extend(cursor.fetchall())
with open("ClassTest.txt", "a", encoding="utf8") as csv_file:
writer = csv.writer(csv_file, delimiter="\t")
writer.write(["Title", "Description", "Category"])
for title, description, category in rows:
writer.writerows([title.replace("\n", " "), description, category])
if __name__ == "__main__":
main()
And the in the other program:
data = pd.read_csv("ClassTest.txt", delimiter="\t")
I am needing to either add or grab column names from a DB file using Sqlite and Xslx Writer.
The code I have (Shown Below) is my attempt on putting column names in the workbook manually. I am unable to figure out how to alter the loop to write on the second row to allow the titles to stay. I am hoping someone can help me figure this out or have a better way to pull the titles directly from my DB file.
I've tried adjusting "write_row(i, 0, row)" to "write_row(1,0,row)". Which causes it to write under the titles but it only grabs the last line of data from the db. I'm aware that the loop needs i somewhere in that, but I have no idea where.
def export():
todays_date = "Log "+str(datetime.datetime.now().strftime("%Y-%m-%d_%H_%M") )+ '.xlsx'
workbook = xlsxwriter.Workbook(todays_date)
worksheet = workbook.add_worksheet()
conn = sqlite3.connect("logging.db")
cur = conn.cursor()
cur.execute("SELECT * FROM ML")
mysel = cur.execute("SELECT * FROM ML")
worksheet.write("A1", 'ID')
worksheet.write("B1", 'Model')
worksheet.write("C1", 'Serial')
worksheet.write("D1", 'Test')
worksheet.write("E1", 'Before')
worksheet.write("F1", 'After')
worksheet.write("G1", 'Duration')
worksheet.write("H1", 'TimeStamp')
for i, row in enumerate(mysel):
worksheet.write_row(i, 0, row)
workbook.close()
os.startfile(todays_date)
i have a prblem with importing CSV-file into Database...
Im using SQLAlchemy in Python and wanted to open a CSV-File than show it in QTableWidget to maybe change the values and after write it to DB (New Table).
def setinTable(self):
colcnt = len(self.title)
rowcnt = len(self.data)
self.tabel_model = QtGui.QTableWidget(rowcnt, colcnt)
vheader = QtGui.QHeaderView(QtCore.Qt.Orientation.Vertical)
self.tabel_model.setVerticalHeader(vheader)
hheader = QtGui.QHeaderView(QtCore.Qt.Orientation.Horizontal)
self.tabel_model.setHorizontalHeader(hheader)
self.tabel_model.setHorizontalHeaderLabels(self.title)
for i in range(rowcnt):
for j in range(len(self.data[0])):
item = QtGui.QTableWidgetItem(str(self.data[i][j]))
self.tabel_model.setItem(i, j, item)
self.tabel_model.horizontalHeader().sectionDoubleClicked.connect(self.changeHorizontalHeader)
self.setCentralWidget(self.tabel_model)
Get CSV-Data
def getdata(filepath):
with open(filepath, 'r') as csvfile:
sample = csvfile.read(1024)
dialect = csv.Sniffer().sniff(sample, [';',',','|'])
csvfile.seek(0)
reader = csv.reader(csvfile,dialect=dialect)
header = next(reader)
lines = []
for line in reader:
lines.append(line)
return lines
Reading and showing the CSV-File data in a QTableWidget is working .. but i dont know how to save it to a MySQL Database
For an easier way to load a csv into a database table, check out the 'odo' python project - https://pypi.python.org/pypi/odo/0.3.2
--
To use a table via SQL Alchemy one approach is to use a session and call "update":
myRow = myTable(
column_a = 'foo',
column_b = 'bar')
myRow.column_c = 1 + 2
mySession.update(myRow)