Is it possible to download data to a csv file by the cx_Oracle module, so that the floating point numbers have a comma instead of a dot?
I need this functionality to properly load the downloaded csv file into another table in the Oracle database. When I try to load such a csv file with floating point numbers, I get an error: cx_Oracle.DatabaseError: ORA-01722: invalid number
I have already solved the problem using the pandas library.
My question:
Is there a solution without the use of data frame pandas.
def load_csv():
conn = cx_Oracle.connect(user=db_user, password=db_userpwd, dsn=dsn, encoding="UTF-8")
cursor = conn.cursor()
cursor.execute(str("select * from tablename"))
result_set = cursor.fetchall()
with open(table_name['schemat']+"__"+table_name['tabela']+".csv", "w") as csv_file:
csv_writer = csv.writer(csv_file, delimiter='|', lineterminator="\n", quoting=csv.QUOTE_NONNUMERIC)
for row in result_set:
csv_writer.writerow(row)
#df = pandas.read_sql("select * from tablename", conn)
#df.to_csv(table_name['schemat']+"__"+table_name['tabela']+".csv", index = False, encoding='utf-8', decimal=',', sep='|', header=False)
cursor.close()
conn.close()
def export_csv():
# Open connection to Oracle DB
conn = cx_Oracle.connect(user=db_user, password=db_userpwd, dsn=dsn, encoding="UTF-8")
# Open cursor to Oracle DB
cursor = conn.cursor()
batch_size = 1
with open(table_name['schemat']+"__"+table_name['tabela']+".csv", 'r') as csv_file:
csv_reader = csv.reader(csv_file, delimiter='|' )
sql = sql_insert
data = []
for line in csv_reader:
data.append([i for i in line])
if len(data) % batch_size == 0:
cursor.executemany(sql, data)
data = []
if data:
cursor.executemany(sql, data)
conn.commit()
cursor.close()
conn.close()
I tried to set it up by changing the session, but unfortunately it doesn't work for me.
# -*- coding: utf-8 -*-
import csv
import os
import sys
import time
import decimal
import pandas as pd
import cx_Oracle
dsn = "(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=xxx)" \
"(PORT=xxx))(CONNECT_DATA=(SERVICE_NAME = xxx)))"
db_user = "xxx"
db_userpwd = "xxx"
def init_session(conn, requested_tag):
cursor = conn.cursor()
cursor.execute("alter session set nls_numeric_characters = ', '")
cursor.execute("select to_number(5/2) from dual")
dual, = cursor.fetchone()
print("dual=", repr(dual))
pool = cx_Oracle.SessionPool(user=db_user, password=db_userpwd,
dsn=dsn, session_callback=init_session, encoding="UTF-8")
with pool.acquire() as conn:
# Open cursor to Oracle DB
cursor = conn.cursor()
cursor.execute("select value from nls_session_parameters where parameter = 'NLS_NUMERIC_CHARACTERS'")
nls_session_parameters, = cursor.fetchone()
print("nls_session_parameters=", repr(nls_session_parameters))
#qryString = "select * from tablename"
#df = pd.read_sql(qryString,conn)
#df.to_csv(table_name['schemat']+"__"+table_name['tabela']+".csv", index = False, encoding='utf-8', decimal=',')
cursor.execute(str("select * from tablename"))
result_set = cursor.fetchall()
#result, = cursor.fetchone()
#print("result is", repr(result))
with open(table_name['schemat']+"__"+table_name['tabela']+".csv", "w") as csv_file:
csv_writer = csv.writer(csv_file, delimiter='|', lineterminator="\n")
for row in result_set:
csv_writer.writerow(row)
I would be grateful for any suggestions on how I can get data to csv file without pandas library.
example:
problematic result: 123.45
correct result: 123,45
Another, possibly simpler option:
Create an output type handler that tells Oracle to fetch the value as a string. Then replace the period with a comma:
import cx_Oracle as oracledb
def output_type_handler(cursor, name, default_type, size, precision, scale):
if default_type == oracledb.DB_TYPE_NUMBER:
return cursor.var(str, arraysize=cursor.arraysize,
outconverter=lambda s: s.replace(".", ","))
conn = oracledb.connect("user/password#host:port/service_name")
conn.outputtypehandler = output_type_handler
with conn.cursor() as cursor:
cursor.execute("select * from TestNumbers")
for row in cursor:
print(row)
Put the output type handler on the cursor if you only want to do this for one query instead of all queries.
You can do by TO_CHAR(<numeric_value>,'999999999D99999999999','NLS_NUMERIC_CHARACTERS=''.,''') conversion such as
cursor.execute("""
SELECT TRIM(TO_CHAR(5/2,'999999999D99999999999',
'NLS_NUMERIC_CHARACTERS=''.,'''))
FROM dual
""")
result_set = cursor.fetchall()
with open(table_name['schemat']+"__"+table_name['tabela']+".csv", "w") as csv_file:
csv_writer = csv.writer(csv_file, delimiter='|', lineterminator="\n")
for row in result_set:
csv_writer.writerow(row)
btw, switching ''.,'' to '',.'' will yield 2,50000000000 again
Since you're writing to a text file and presumably also want to avoid any Oracle decimal format to Python binary format precision issues, fetching as a string like Anthony showed has advantages. If you want to move the decimal separator conversion cost to the DB you could combine his solution and yours by adding this to your original code:
def output_type_handler(cursor, name, default_type, size, precision, scale):
if default_type == cx_Oracle.NUMBER:
return cursor.var(str, arraysize=cursor.arraysize)
and then after you open the cursor (and before executing), add the handler:
cursor.outputtypehandler = output_type_handler
Since the DB does the conversion to string, the value of NLS_NUMERIC_CHARACTERS is respected and you get commas as the decimal separator.
Im new to SQL and PostgreSQL and I cant understand whats going on with this code.
Im trying to insert csv to postgres with this code:
import csv
import psycopg2 as pg
filename = 'myfile.csv'
try:
conn = pg.connect(user="myuser",
password="mypass",
host="myhost",
port="5432",
database="mydb")
cursor = conn.cursor()
with open(filename, 'r') as f:
reader = csv.reader(f)
next(reader) # This skips the 1st row which is the header.
for record in reader:
print(record)
cursor.execute("""INSERT INTO auth VALUES (%s, %s, %s, %s)""", record)
conn.commit()
except (Exception, pg.Error) as e:
print(e)
finally:
if (conn):
cursor.close()
conn.close()
print("Connection closed.")
but it raise error insert has more target columns than expressions
LINE 1: ...00000000-0000-0000-0000-000000000000', '1580463062', 'auto')
but here is what i want to insert
['00000000-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000000', '1580463062', 'auto']
and its look like its definitely has exactly 4 colums
I also tried to change the encoding of csv from ASCII to UTF-8 and UTF-8_SIG but i still get this error
I solve my problem with this code
import psycopg2
conn = psycopg2.connect("host=host dbname=dbname user=user
password=password")
cur = conn.cursor()
with open(filename, 'r') as f:
next(f)
cur.copy_from(f, 'auth', sep=',')
conn.commit()
cur.close()
conn.close()
I've accidentally created an infinite loop importing a CSV file using python. I don't see where the error is!
This is my code:
csv_reader = csv.reader(csv_file, delimiter=',') # Read the file
next(csv_reader) # Skip the header row
insert_sql = "INSERT INTO billing_info (InvoiceId, PayerAccountId) VALUES (%s, %s, )" # Insert SQL statement
print("Importing Data")
for row in csv_reader:
print(row)
cursor.execute(insert_sql, row)
mydb.commit()
The file is 4GB in size, so it's huge. However, in the print statement, I can tell that it starts over again at the top of the file.
Here is the entirety of the function:
mydb = mysql.connector.connect(user='xxxx', password='xxx',
host='xxxx',
database='aws_bill')
cursor = mydb.cursor()
def read_csv_to_sql(source):
try:
with open(source) as csv_file: # Open the file
csv_reader = csv.reader(csv_file, delimiter=',') # Read the file
next(csv_reader) # Skip the header row
insert_sql = "INSERT INTO billing_info (InvoiceId, PayerAccountId) VALUES (%s, %s)" # Insert SQL statement
print("Importing Data")
for row in csv_reader:
print(row)
cursor.execute(insert_sql, row)
mydb.commit()
print("Done importing data.")
except Exception as e:
print("Exception:", e)
mydb.rollback()
finally:
mydb.close()
Why is are these statements creating an infinite loop?
I am beginner to python, and I have this code to decompress log file, but I have the error Invalid argument. I don't know why I got this error, in my opinion, I think it is because the log file is too big, cause I am scanning a log file which is 2gb file. But I have no idea how to fix the error. Please help, thank you. And below is my code with the error:
import glob
import gzip
import os
import pymysql
import logging
# path to gz directory
GZ_DIR = '/Users/kiya/Desktop/mysql/csv'
# Database Infomation
DB_HOST='locahost'
DB_USER='dbuser'
DB_PASS='dbPassword'
DB_NAME='dbname'
LOGFILE="exception.log"
def csv_reader(file, header=False):
import csv
with open(file, "r") as f:
reader = csv.reader(f)
if header:
next(reader)
for row in reader:
yield row
def import_sql(filename, dbHostName, dbUser, dbPassword, databaseName):
db = pymysql.connect(host=dbHostName,
user=dbUser,
password=dbPassword,
db=databaseName,
charset='utf8')
for row in csv_reader(filename, False):
# prepare a cursor object using cursor() method
with db.cursor() as cursor:
if row[3] == "THREAT" and row[4] == "url":
sql = ("INSERT INTO PADIAGDB.url ("
"Domain,Receive_Time,Serial,Type,Threat_Content_Type,"
"Config_Version,Generate_Time,Source_address,Destination_address,"
"NAT_Source_IP,NAT_Destination_IP,Rule,Source_User,"
"Destination_User,Application,Virtual_System,Source_Zone,"
"Destination_Zone,Inbound_Interface,Outbound_Interface,Log_Action,"
"Time_Logged,Session_ID,Repeat_Count,Source_Port,Destination_Port,"
"NAT_Source_Port,NAT_Destination_Port,Flags,IP_Protocol,Action,"
"URL_Filename,Threat_Content_Name,Category,Severity,Direction,"
"Sequence_Number,Action_Flags,Source_Country,Destination_Country,"
"cpadding,contenttype,pcap_id,filedigest,cloud,url_idx,user_agent,"
"filetype,xff,referer,sender,subject,recipient,reportid,"
"dg_hier_level_1,dg_hier_level_2,dg_hier_level_3,dg_hier_level_4,"
"Virtual_System_Name,Device_Name,file_url )"
""
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,"
"%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,"
"%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s"
"); ")
elif row[3] == "SYSTEM":
sql = ("INSERT INTO PADIAGDB.system ("
"Domain,Receive_Time,Serial,Type,Threat_Content_Type,Config_Version,"
"Generate_Time,Virtual_System,Event_ID,Object,fmt,id,module,Severity,"
"Description,Sequence_Number,Action_Flags,dg_hier_level_1,"
"dg_hier_level_2,dg_hier_level_3,dg_hier_level_4,Virtual_System_Name,"
"Device_Name )"
""
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,"
"%s,%s,%s );")
else:
continue
try:
cursor.execute('SET foreign_key_checks = 0')
# Execute the SQL command
r = cursor.execute(sql, row)
#Commit your changes in the database
cursor.execute('SET foreign_key_checks = 1')
db.commit()
except Exception as e:
logging.exception(e)
db.rollback()
# disconnect from server
db.close()
gz_files = (gz for gz in glob.glob(os.path.join(GZ_DIR, '*.gz')))
for gz_file in gz_files:
with gzip.open(gz_file, 'rb') as in_file:
s = in_file.read()
sql_file = gz_file[:-3]
sql_file = sql_file[:-4] + '.csv'
with open(sql_file, 'wb') as out_file:
out_file.write(s)
import_sql(out_file, DB_HOST, DB_USER, DB_PASS, DB_NAME)
os.remove(sql_file)
This is the error I got:
Traceback (most recent call last):
File "/Users/kiya/Desktop/mysql/csv/sql3.py", line 180, in <module>
out_file.write(s)
OSError: [Errno 22] Invalid argument
for reading big files, you will have to read and write in chucks, try smth like (draft)
fr = open(input_file, 'rb')
fw = open(output_file, 'wb')
while True:
chunk = fr.read(1024)
if not chunk:
break
fw.write(chunk)
fr.close()
fw.close()
you can use context mangers of course aka with
I would like to load a csv into a sqlite. My CSV contains utf-8 characters like (é, à, ü, ♀...).
These characters are displayed as ’ or é in my sqlite. I have used con.text_factory = str but it doesn't change anything.
I have also tried .decode('utf8') as this question suggests but I get the error: 'str' object has no attribute 'decode'
import sqlite3
import csv
import os
db= "mydb.sqlite"
con = sqlite3.connect(db)
con.text_factory = str # allows utf-8 data to be stored
cursor = con.cursor()
csvfile= 'mycsv.csv'
tablename = os.path.splitext(os.path.basename(csvfile))[0]
with open (csvfile, 'r') as f:
reader = csv.reader(f)
header_line_from_csv = next(reader)
columns = [h.strip() for h in header_line_from_csv] #Strips white space in header
headers= ', '.join([f'{column} text' for column in columns]) ##
sql = f'CREATE TABLE {tablename} ({headers})'
print (sql)
cursor.execute(sql)
query = 'insert into {0}({1}) values ({2})'
query = query.format(tablename, ','.join(columns), ','.join('?' * len(columns)))
print(query)
cursor = con.cursor()
for row in reader:
cursor.execute(query, row)
con.commit()
print(cursor.rowcount)
cursor.execute(f"SELECT * FROM {tablename}")
print("fetchall:\n",cursor.fetchall())
print(cursor.description)
con.close()
You can add this line at beginning:
# -*- coding:utf-8 -*-
to guarantee that your script will use this encoding. Try also to open csv file with utf-8 encoding:
with open (csvfile, 'r', encoding='utf-8') as f: