Develop Reference

sqlite3.OperationalError: no such column: Python and Sqllite3

My Code:
From Python 3.8
def email_address_grab(email_list):
""" This function takes in a list of emails and puts them into a sql database"""
#import module
import sqlite3 as sql
#Setup sql
#create connection for sql
connection = sql.connect("emailList.db")
#create cursor
crsr = connection.cursor()
#create sql table
cmd = """CREATE TABLE emails (
email_handle TEXT,
email_domain VARCHAR(20));"""
crsr.execute(cmd)
#iterate through email list
index = 0
for email in email_list:
#split email with a delimiter of "#"
email_list[index] = email.split('#')
index += 1
#while loop to put all data into table
ct = 0
while ct <= (len(email_list) - 1):
for i in range(0, len(email_list)):
for j in range(0, len(email_list)):
email_address_1 = email_list[i]
email_address_2 = email_list[j]
cmd = f"""INSERT INTO emails (email_handle, email_domain) VALUES ({email_address_1}, {email_address_2});"""
crsr.execute(cmd)
ct += 1
#get the contents of the table
crsr.execute("SELECT * FROM emails;")
#store contents in a variable
email_address_list = crsr.fetchall()
#save changes to sql table
connection.commit()
#close connection
connection.close()
#return print statement for data
return print(email_address_list)
Error:
Traceback (most recent call last):
File "c:/Users/USER/Desktop/email grabber.py", line 79, in
email_address_grab(["testemail123#gmail.com"])
File "c:/Users/USER/Desktop/email grabber.py", line 58, in email_address_grab
crsr.execute(cmd)
sqlite3.OperationalError: no such column: 'testemail123', 'gmail.com'

Your issue is because this is your final command string:
"""INSERT INTO emails (email_handle, email_domain) VALUES (testemail123, gmail.com);"""
while what is a valid string is:
"""INSERT INTO emails (email_handle, email_domain) VALUES ("testemail123", "gmail.com");"""
So you should be using something like:
cmd = f"""INSERT INTO emails (email_handle, email_domain) VALUES ('{email_address_1}', '{email_address_2}');"""
Although, with sqlite3, you should be passing in parameters with the execute call. This is to help prevent sql injection attacks because the way that you are using formatted strings can result in catastrophic attacks.
You should pass parameters to sqlite3 instances like this:
cmd = """INSERT INTO emails (email_handle, email_domain) VALUES (?, ?);"""
crsr.execute(cmd, (email_address_1, email_address_2))

'NoneType' error using Python SSHTunnelForwarder

Im trying to put together a script that will run on a scheduled basis, updating a database located on the cloud. im using SSHTunnelForwarding to establish a connection with my database. its all packed within a function so that all i have to do is pass the query string within the function and it runs. this function is looping over about 1000 rows of data within a CSV. each row is being altered to the proper format and a sql expression is being created.
with open (CSV, 'r') as f:
reader = csv.reader(f)
data = next(reader)
#load new data
for newrow in reader:
row = []
for val in newrow:
try:
val = float(val)
except:
val = "'"+val.strip()+"'"
row.append(val)
id = row[0]
phase_order = row[1]
Ajera_Project_Key = row[2]
project_id = row[3]
Project_Description = row[4]
Ajera_Client_Key = row[5]
Client = row[6]
Ajera_PM_Key = row[7]
Project_Manager = row[8]
Ajera_PIC_Key = row[9]
Principal_In_Charge = row[10]
title = row[11]
Ajera_Dept_Key = row[12]
Project_Status = row[13]
Phase_Status = row[14]
Department = row[15]
Project_Type = row[16]
start = row[17]
end = row[18]
hours_budgeted = row[19]
Hours_Worked = row[20]
Hours_Remaining = row[21]
Total_Contract_Amount = row[22]
Billed = row[23]
Billed_Labor = row[24]
Billed_Hours = row[25]
WIP = row[26]
Spent = row[27]
Spent_Labor = row[28]
FTEs = row[29]
q = 'INSERT INTO project_phases_test VALUES ({0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13},{14},{15},{16},{17},{18},{19},{20},{21},{22},{23},{24},{25},{26},{27},{28},{29});'.format(id,phase_order,Ajera_Project_Key,project_id,Project_Description,Ajera_Client_Key,Client,Ajera_PM_Key,Project_Manager,Ajera_PIC_Key,Principal_In_Charge,title,Ajera_Dept_Key,Project_Status,Phase_Status,Department,Project_Type,start,end,hours_budgeted,Hours_Worked,Hours_Remaining,Total_Contract_Amount,Billed,Billed_Labor,Billed_Hours,WIP,Spent,Spent_Labor,FTEs)
print (q)
query(q)
the issue im having is when i try to enter a query string other than a simple select. every insert, update, delete statement i pass to it, i get an error that says 'NoneType' object is not iterable. however, as i mentioned before, if i pass a select statement, i get the result im looking for and i dont get any errors.
here is the function
def query(q):
try:
with SSHTunnelForwarder(
(host, 22),
ssh_username=ssh_username,
ssh_password=ssh_password,
ssh_private_key=ssh_private_key,
remote_bind_address=(localhost, 3306)
) as server:
conn = db.connect(host=localhost,
port=server.local_bind_port,
user=user,
passwd=password,
db=database)
pd.read_sql_query(q, conn)
conn.close()
return
except Exception as e:
print e
pass
the error is coming from the print e statement.
full error
Traceback (most recent call last):
File "C:\MAMP\htdocs\WIGHTcloud\dataLoader\load\loader.py", line 183, in <module>
query(drop) # drop phases table
File "C:\MAMP\htdocs\WIGHTcloud\dataLoader\load\loader.py", line 140, in query
pd.read_sql_query(q, conn)
File "C:\Python27\ArcGIS10.6\lib\site-packages\pandas\io\sql.py", line 431, in read_sql_query
parse_dates=parse_dates, chunksize=chunksize)
File "C:\Python27\ArcGIS10.6\lib\site-packages\pandas\io\sql.py", line 1600, in read_query
columns = [col_desc[0] for col_desc in cursor.description]
TypeError: 'NoneType' object is not iterable

using what nilleb said, i plugged it into the sshTunnelForwarder and got it working.
with sshtunnel.SSHTunnelForwarder(ssh_address_or_host = ssh_host,
ssh_username = ssh_username,
ssh_password = ssh_password,
ssh_pkey = ssh_pkey,
remote_bind_address=('localhost', 3306),
) as tunnel:
mydb = mysql.connector.connect(
host=localhost,
user=sql_username,
passwd=sql_password,
database=db_name,
port=tunnel.local_bind_port
)
mycursor = mydb.cursor()
sql = "INSERT INTO customers (name, address) VALUES (%s, %s)"
val = ("John", "Highway 21")
mycursor.execute(sql, val)
mydb.commit()
print(mycursor.rowcount, "record inserted.")

I would say that this is expected. An instruction like SELECT will produce an output, including the table headers.
While other DDL statements do not produce an output and thus raise an exception in the pd.read_sql_query function.
You should rather consider retrieving a cursor, and executing the DDL statement on it, like
mycursor = conn.cursor()
sql = "INSERT INTO customers (name, address) VALUES (%s, %s)"
val = ("John", "Highway 21")
mycursor.execute(sql, val)
More info here: https://www.w3schools.com/python/python_mysql_insert.asp

AttributeError: 'Engine' object has no attribute 'conn'

So I am trying to create an auto update to SQL from another excel file, by unique value, as to know what is the new data to add to the database..
There's different in columns names between the database and the excel file as in the database and names without spaces...
I tried to do it with pandas it gave me the same error
So here's my simple code tried with xlrd
import xlrd
from sqlalchemy import create_engine
def insert():
book = xlrd.open_workbook(r"MNM_Rotterdam_5_Daily_Details-20191216081027 - Copy (2).xlsx")
sheet = book.sheet_by_name("GSM Details")
database = create_engine(
'mssql+pyodbc://WWX542337CDCD\SMARTRNO_EXPRESS/myDB?driver=SQL+Server+Native+Client+11.0') # name of database
cnxn = database.raw_connection
cursor = cnxn.cursor()
query = """Insert INTO [myDB].[dbo].[mnm_rotterdam_5_daily_details-20191216081027] (Date, SiteName, CellCI, CellLAC, CellName, CellIndex) values (?,?,?,?,?,?)"""
for r in range(1, sheet.nrows):
date = sheet.cell(r,0).value
site_name = sheet.cell(r,3).value
cell_ci = sheet.cell(r,4).value
cell_lac = sheet.cell(r,5).value
cell_name = sheet.cell(r,6).value
cell_index = sheet.cell(r,7).value
values = (date, site_name, cell_ci, cell_lac, cell_name, cell_index)
cursor.execute(query, values)
cnxn.commit()
# Close the cursor
cursor.close()
# Commit the transaction
database.commit()
# Close the database connection
database.close()
# Print results
print ("")
print ("")
columns = str(sheet.ncols)
rows = str(sheet.nrows)
print ("Imported", columns,"columns and", rows, "rows. All Done!")
insert()
and this is the error:
I tried to change the range I found another error:
Traceback (most recent call last):
File "D:/Tooling/20200207/uniquebcon.py", line 48, in <module>
insert()
File "D:/Tooling/20200207/uniquebcon.py", line 37, in insert
database.commit()
AttributeError: 'Engine' object has no attribute 'commit'
I think this is related to SQL-Alchemy in the connection

Instead of creating the cursor directly with
cursor = database.raw_connection().cursor()
you can create a connection object, then create the cursor from that, and then call .commit() on the connection:
cnxn = database.raw_connection()
crsr = cnxn.cursor()
# do stuff with crsr ...
cnxn.commit()

Importing a csv file to sqllite3 using python functional programming

I know there are some other posts out there, but I was not able to find the specific question I had in mind.
I'm using US_baby_names csv file. and want to import this csv file line by line into sqlite3 as a table.
I'm able to create the table called storage.
I'm then trying to read lines in the csv file and put it into that table, but I must be doing something wrong.
import sqlite3 as sql
from sqlite3 import Error
import csv
def CreateConnection ( dbFileName ):
try:
conn = sql.connect(dbFileName)
return conn
except Error as e:
print(e)
return None
def CreateNew( dbConnection, new):
sql = """INSERT INTO storage (dat, Id, Name, Year, group, subgroup, Count)
VALUES (?,?,?,?,?,?,?)"""
try:
cursor = dbConnection.cursor()
cursor.execute(sql, new)
return cursor.lastrowid
except Error as e:
print(e)
def Main():
database = "storage.db"
dbConnection = CreateConnection(database)
with open('storage.csv', 'rb') as fin:
dr = csv.DictReader(fin)
to_db = [(i['dat'], i['Id'], i['Name'], i['Year'], i['group'], i['subgroup'], i['Count']) \
for i in dr]
cursor.executemany(CreateNew(sql, to_db))
dbConnection.close()
if __name__ == "__main__":
Main()
I believe my cursor.executemany is wrong, but I'm not able to figure out what else to do..
Thanks

You are almost right with much of your code, but:
in cursor.execute(sql, new) you are passing an iterable, new, to sqlite3.execute() (which requires a simple SQL statement), instead of sqlite3.executemany().
Moreover, the result of CreateNew() is an integer, lastrowid, and you pass that result to executemany().
You must use Connection.commit() to save the changes to the database, and Connection.rollback() to discard them.
You must open the file for the csv.DictReader class as a text file, in r or rt mode.
Finally, remember that sqlite3.Connection is a context manager, so you can use it in a with statement.
This should be your desired outcome:
import sqlite3 as sql
from sqlite3 import Error
import csv
def create_table(conn):
sql = "CREATE TABLE IF NOT EXISTS baby_names("\
"dat TEXT,"\
"Id INTEGER PRIMARY KEY,"\
"Name TEXT NOT NULL,"\
"Year INTEGER NOT NULL,"\
"Gender TEXT NOT NULL,"\
"State TEXT NOT NULL,"\
"Count INTEGER)"
conn.execute(sql)
conn.execute("DELETE FROM baby_names")
def select_all(conn):
for r in conn.execute("SELECT * FROM baby_names").fetchall():
print(r)
def execute_sql_statement(conn, data):
sql = "INSERT INTO baby_names "\
"(dat, Id, Name, Year, Gender, State, Count) "\
"VALUES (?,?,?,?,?,?,?)"
try:
cursor = conn.executemany(sql, data)
except Error as e:
print(e)
conn.rollback()
return None
else:
conn.commit()
return cursor.lastrowid
def main():
with sql.connect('baby_names.db') as conn, open('US_Baby_Names_right.csv', 'r') as fin:
create_table(conn)
dr = csv.DictReader(fin)
data = [(i['dat'], i['Id'], i['Name'], i['Year'], i['Gender'], i['State'], i['Count']) for i in dr ]
lastrowid = execute_sql_statement(conn, data)
select_all(conn)
main()
I added a create_table() function just to test my code. I also made up a sample test file as follows:
dat,Id,Name,Year,Gender,State,Count
1,1,John,1998,M,Washington,2
2,2,Luke,2000,M,Arkansas,10
3,3,Carrie,1999,F,Texas,3
The output of the select_all() function is:
('1',1,'John',1998,'M','Washington',2)
('2',2,'Luke',2000,'M','Arkansas',10)
('3',3,'Carrie',1999,'F','Texas',3)

Why is this query not working from python to SQL?

I am new to coding and databases, I can not get the query to work if I write it long hand but I have a lot to carry out and want it in a function but cannot get it to work, it returns a parameters error
import mysql.connector
def connection_check_1(query, value):
mydb = mysql.connector.connect(
host="******",
user="*****",
passwd="*****",
database="****"
)
mycursor = mydb.cursor()
mycursor.execute(query, (value))
myresult = mycursor.fetchall()
mydb.close()
return myresult
value = "sheep"
query = 'select inlicence from licence where animal = %s'
myresult = connection_check_1(query, value)
print(myresult)
Here is the SQL table I have
create table licence
(
animal varchar (20) primary key,
inlicence int (1)
);
This is the error I get
Traceback (most recent call last):
File "*******************", line 20, in
myresult = connection_check_1(query, value)
File "********************", line 13, in connection_check_1
mycursor.execute(query, (value))
File "********************************************88", line 246, in execute
prepared = self._cnx.prepare_for_mysql(params)
File "/home/kev/PycharmProjects/test bed/venv/lib/python3.5/site-packages/mysql/connector/connection_cext.py", line 535, in prepare_for_mysql
raise ValueError("Could not process parameters")
ValueError: Could not process parameters
I have tried changing the way the query is written, changing it to fetchall().

Wrapping a value with () doesn't turn it in to a tuple. You probably meant to add a comma there:
mycursor.execute(query, (value,))
# Creates a one-element tuple-^