using python,I am looping through csv file to read data, then I am ding some modifications on the readied row and call a save function to insert the modified data into MySQL.
def save(Id, modifiedData,):
try:
mydb = mysql.connector.connect(host="localhost",user="use",password="pass",database="data")
sql = "INSERT INTO data (Id, modifiedData) VALUES (%s, %s)"
recordTuple = (Id, modifiedData)
mycursor = mydb.cursor()
mycursor.execute(sql,recordTuple)
mydb.commit()
print("Record inserted successfully into table")
except mysql.connector.Error as error:
print("Failed to insert into MySQL table {}".format(error))
def main():
for row in csv:
#modify row
#creat Id
save(Id, modifiedData,)
but I don't think this is good solution to do MYSQL connection and insert data with each iteration, it will be time and resources consuming , specially when I move to real server in production
how can I improve my solution?
Ideally, connections should be managed by connection pool, should be committed bulky. But amount of csv at most, need not to mind so much. Anyway, If you don't wanna bother it, I recommend using ORM like SQLAlchemy.
You only need to create the connection once, and that should be in function main, who then passes the connection to function save as follows:
def save(mydb, Id, modifiedData):
try:
sql = "INSERT INTO data (Id, modifiedData) VALUES (%s, %s)"
recordTuple = (Id, modifiedData)
mycursor = mydb.cursor()
mycursor.execute(sql,recordTuple)
mydb.commit()
print("Record inserted successfully into table")
except mysql.connector.Error as error:
print("Failed to insert into MySQL table {}".format(error))
def main():
try:
mydb = mysql.connector.connect(host="localhost",user="use",password="pass",database="data")
except mysql.connector.Error as error:
print("Failed to create connection: {}".format(error))
return
for row in csv:
#modify row
#creat Id
save(mydb, Id, modifiedData)
For perhaps even greater performance you can try executemany:
def save(mydb, modified_records):
try:
sql = "INSERT INTO data (Id, modifiedData) VALUES (%s, %s)"
mycursor = mydb.cursor()
mycursor.executemany(sql, modified_records)
mydb.commit()
print("Records inserted successfully into table")
except mysql.connector.Error as error:
print("Failed to insert into MySQL table {}".format(error))
def main():
try:
mydb = mysql.connector.connect(host="localhost",user="use",password="pass",database="data")
except mysql.connector.Error as error:
print("Failed to create connection: {}".format(error))
return
modified_records = []
for row in csv:
#modify row
#creat Id
modified_records.append([id, modifiedData])
save(mydb, modified_records)
Related
error:
You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '%s)' at line 1
upi = upi_entry.get()
mysqldb = mysql.connector.connect(
host="localhost",
user="root",
password="deol9646",
database="train_login",
)
mycursor = mysqldb.cursor()
try:
mycursor.execute(
"""create table if not exists upi_data(upi text)"""
)
sql = "INSERT INTO UPI_DATA (UPI) VALUES (%s)"
val = upi
mycursor.execute(sql, val)
mysqldb.commit()
lastid = mycursor.lastrowid
messagebox.showinfo("information", "upi inserted successfully...")
upi_entry.delete(0, END)
upi_entry.focus_set()
except Exception as e:
print(e)
mysqldb.rollback()
mysqldb.close()
The parameters need to be a tuple; you're passing in val as a single value, so the MySQL driver doesn't turn %s into anything and that ends up a syntax error.
Add a comma to make a parenthesized expression ((upi)) into a 1-tuple: (upi,)
sql = "INSERT INTO UPI_DATA (UPI) VALUES (%s)"
val = (upi,)
I am new to Python and started off with sqlite.
I have two csv transaction.csv and users.csv from where I am reading the data and writing to the sqlite database.Below is the snippet
import csv
import sqlite3 as db
def readCSV_users():
with open('users.csv',mode='r') as data:
dr = csv.DictReader(data, delimiter=',')
users_data = [(i['user_id'], i['is_active']) for i in dr if i['is_active']=='True']
#---------------------
return users_data
def readCSV_transactions():
with open('transactions.csv',mode='r') as d:
dr = csv.DictReader(d, delimiter=',')
trans_data = [(i['user_id'], i['is_blocked'],i['transaction_amount'],i['transaction_category_id']) for i in dr if i['is_blocked']=='False']
#---------------------
return trans_data
def SQLite_connection(database):
try:
# connect to the database
conn = db.connect(database)
print("Database connection is established successfully!")
conn = db.connect(':memory:')
print("Established database connection to a database\
that resides in the memory!")
cur = conn.cursor()
return cur,conn
except exception as Err:
print(Err)
def dbQuery(users_data,trans_data,cur,conn):
try:
cur.executescript(""" CREATE TABLE if not exists users(user_id text,is_active text);
CREATE TABLE if not exists transactions(user_id text,is_blocked text,transaction_amount text,transaction_category_id text);
INSERT INTO users VALUES (?,?),users_data;
INSERT INTO transactions VALUES (?,?,?,?),trans_data""")
conn.commit()
a=[]
rows = curr.execute("SELECT * FROM users").fetchall()
for r in rows:
a.append(r)
return a
except Err:
print(Err)
finally:
conn.close()
if __name__ == "__main__":
database='uit'
users_data=readCSV_users()
trans_data=readCSV_transactions()
curr,conn=SQLite_connection(database)
print(dbQuery(users_data,trans_data,curr,conn))
But I am facing below error.I believe the ? is throwing the error in executescript
cur.executescript(""" CREATE TABLE if not exists users(user_id text,is_active text);
sqlite3.OperationalError: near "users_data": syntax error
Any pointers to resolve this?
Putting users_data directly in query is wrong. It treats it as normal string.
But it seems executescript can't use arguments.
You would have to put values directly in place of ?.
Or you have to use execute()
cur.execute("INSERT INTO users VALUES (?,?);", users_data)
cur.execute("INSERT INTO transactions VALUES (?,?,?,?)", trans_data)
I have several CSV files that I'm trying to upload to a PostgreSQL database.
My current file/function setup works perfectly fine for files without NULLs, but it's when I have null values that I hit an issue.
In the CSV, those nulls are currently empty cells. The function I'm using to upload them to the database looks like this:
insert_query_string = ("""
INSERT INTO sample_table (
primaryKey,
color,
date,
place,
numeric1,
numeric2,
numeric3)
VALUES (%s, %s, %s, %s, %s, %s, %s)
""")
def loadData(cur, conn, query):
"""
Loads data blahblahblah
"""
try:
data_csv = csv.reader(open(data_path + 'data.csv'))
header = next(data_csv)
for row in data_csv:
print(row)
cur.execute(query, row)
print(' - data loaded.')
conn.commit()
except FileNotFoundError:
print("CSV file not found.")
def main():
conn = None
try:
# connect
print("Connecting ... ")
conn = pg.connect(**nfl_params)
cur = conn.cursor()
# load
print("Loading data ... ")
loadData(cur, conn, insert_query_string)
except (Exception, pg.DatabaseError) as error:
print(error)
finally:
if conn is not None:
conn.close()
print("Database Connection closed.")
if __name__ == "__main__":
main()
Some of the numeric columns are null, and yet I can't seem to figure out how to get them into the database. As is, I'm getting invalid input syntax for type numeric: "" so I thought it was trying to read it as a string or something.
This stackoverflow question seems to be saying to replace the empty cells to 'null' or 'NULL' but that doesn't seem to work either.
My problem, as far as I can tell, is because csv's don't have anyway to store null values other than empty, which isn't working for me. I'm thinking for now that I'll replace the null values with -1 or something, but this doesn't seem to be the cleanest way to do it.
Thanks in advance!
So here is my code. I don't know why insert isn't working. A select statement works. It doesn't fail the try catch either leading me to believe the query is executing. Also entering the insert query manually into MySQL Workbench seems to work fine.
def runQuery(query):
try:
conn = mysql.connector.connect(host='localhost',
database='optionsdata',
user='python',
passwd='python')
cursor = conn.cursor()
cursor.execute(query)
conn.close()
cursor.close()
print(query)
except Error as e:
print("Error", e)
def convertDate(date_str):
date_object = datetime.datetime.strptime(date_str, '%m/%d/%Y').date()
return date_object
ticker = "MSFT"
html = urlopen("https://api.nasdaq.com/api/quote/" + ticker + "/option-chain?assetclass=stocks&todate=2020-05-08&fromdate=2020-04-07&limit=0").read().decode('utf-8')
optionsData = json.loads(html)
rows = optionsData["data"]["optionChainList"]["rows"]
for row in rows:
call = row["call"]
expiryDate = convertDate(call["expiryDate"])
query = "INSERT INTO `optionsdata`.`call` (`ticker`, `symbol`, `last`, `change`, `bid`, `ask`, `volume`, `openinterest`, `strike`, `expiryDate`, `grabTime`) VALUES ('{0}', '{1}', '{2}', '{3}', '{4}', '{5}', '{6}', '{7}', '{8}', '{9}', '{10}');".format(ticker, call["symbol"], call["last"], call["change"], call["bid"], call["ask"], call["volume"], call["openinterest"], call["strike"], expiryDate, datetime.datetime.now())
runQuery(query)
A sample of what an insert query looks like
INSERT INTO `optionsdata`.`call` (`ticker`, `symbol`, `last`, `change`, `bid`, `ask`, `volume`, `openinterest`, `strike`, `expiryDate`, `grabTime`) VALUES ('MSFT', '#MSFT 200508C00175000', '3.21', '-0.29', '2.80', '4.25', '54', '228', '175.00', '2020-05-08', '2020-04-09 19:39:22.554538');
This is a great question! I spent hours trying to figure this out a few weeks ago. It's tricky because after executing the query, you have to call
conn.commit()
to actually update the data. So change your runQuery function like this:
def runQuery(query):
try:
conn = mysql.connector.connect(host='localhost',
database='optionsdata',
user='python',
passwd='python')
cursor = conn.cursor()
cursor.execute(query)
conn.commit() # Added commit line
conn.close()
cursor.close()
print(query)
except Error as e:
print("Error", e)
See this doc page for more info.
I have this code and i need to get the lastrowid as a return statement. How can i fix it
def main():
while True:
#code here
for item in name2:#break
conn = sqlite3.connect("foods.db")
cursor = conn.cursor()
cursor.execute("INSERT INTO INPUT33 (NAME) VALUES (?);", (name2,))
cursor.execute("select MAX(rowid) from [input33];")
conn.commit()
conn.close()
for rowid in cursor:break
for elem in rowid:
return rowid#this is not working
print(m)
You closed the database, so any cursor no longer has access to the data. Retrieve the data before closing. I am assuming here that you have a reason to re-open the database in a loop here.
def main():
while True:
for item in name2:
conn = sqlite3.connect("foods.db")
cursor = conn.cursor()
with conn:
cursor.execute("INSERT INTO INPUT33 (NAME) VALUES (?);", (name2,))
cursor.execute("select MAX(rowid) from [input33];")
rowid = cursor.fetchone()[0]
conn.close()
return rowid