Develop Reference

Insert sqlite3 from variables

i've been trying to create a database, the column names of my table would come from a list:
import sqlite3
L = ["Nom","Age","Taille"]
list2 = ["Karl", "11", "185"]
M = []
R = 0
con = sqlite3.connect(":memory:")
con.execute("CREATE TABLE master ("+",".join(L)+")")
Then Either :
for e in L:
R += 1
con.execute("INSERT INTO master("+",".join(L)+") VALUES (?,?,?)",list2[R-1])
or
for e in L:
R += 1
con.execute("INSERT INTO master(e) VALUES (?)",list2[R-1])
or
listX=[list2[0],list2[1],list2[3])
con.executemany("INSERT INTO master ("+",".join(L)+") VALUES ("+",".join(M)+")", (listX))

Check the documentation: https://docs.python.org/3.8/library/sqlite3.html
In your case:
import sqlite3
con = sqlite3.connect(":memory:")
columns = ["Nom", "Age", "Taille"]
columns_str = '"' + '","'.join(columns) + '"'
con.execute(f"CREATE TABLE people ({columns_str})")
data = [
('Karl', 11, 185)
]
stmt = f"INSERT INTO people ({columns_str}) VALUES (?, ?, ?)"
con.executemany(stmt, data)
Also, probably don't call your table master - that'll get very confusing later. Names like L and list2 also don't help. Be clear in naming your variables, name them after what they mean or contain. Future you will thank you.
A little bit cleaner perhaps:
import sqlite3
con = sqlite3.connect(":memory:")
columns = ("Nom", "Age", "Taille")
con.execute("CREATE TABLE people (%s, %s, %s)" % columns)
data = [
('Karl', 11, 185)
]
stmt = f"INSERT INTO people (%s, %s, %s) VALUES (?, ?, ?)" % columns
con.executemany(stmt, data)

Python 3 MySQL find number of rows error?

I have tried several variation of this, and for that reason I am coming here for guidance. Where is this incorrect?
with connection.cursor() as cur:
sql = 'select * from table where var1 = %s, var2 = %s, var3 = %s, var4 = %s, var5 = %s'
cur.execute(sql, (var1val, var2val, var3val, var4val, var5val))
connection.commit()
l_fetch = cur.fetchall()
rc = int(l_fetch.rowcount)
print('rc len lerr_log: ' + rc)
if(rc > 0):
#result found

cur.fetchall() returns a list, it doesn't have a rowcount attribute.
The number of rows is cur.rowcount and len(l_fetch).

Running a python with MySQL insert command getting an error on parameters

I am trying to do an insert from Python to MySQL and I am getting
mysql.connector.errors.ProgrammingError: Not enough parameters for the SQL statement;
I have had a bit of a look online and I know its got something to do with tuble but I can't work out what to change, I have looked around and at my code, there are 10 items in the responce_data, 10 in the SQL and I have 10 %s so I don't know where I am going wrong
import urllib.parse
import requests
import mysql.connector
mydb = mysql.connector.connect(
host="localhost",
user="root",
passwd="**",
database="flightdata"
)
mycursor = mydb.cursor()
main_api = 'https://www.sydneyairport.com.au/_a/flights/?query=&flightType=departure&terminalType=domestic&date=2019-11-10&sortColumn=scheduled_time&ascending=true&showAll=true'
address = 'lhr'
url = main_api + urllib.parse.urlencode({address: address})
response_data = requests.get(url).json()
for element in response_data['flightData']:
flight_id = element['id']
airline = element['airline']
destination = element['destinations']
flightNumbers = element['flightNumbers']
scheduledTime = element['scheduledTime']
estimatedTime = element['estimatedTime']
scheduledDate = element['scheduledDate']
latestTime = element['latestTime']
status = element['status']
statusColor = element['statusColor']
sql = "INSERT INTO flightinfo (id, airline, destinations, flightNumbers, scheduledTime, estimatedTime, scheduledDate, latestTime, status, statusColor) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
#sql = "INSERT INTO flightinfo (flight_id, airline, destination, flightNumbers, scheduledTime, estimatedTime, estimatedTime, scheduledDate, latestTime, status, statusColor ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s ,%s))"
val = [(flight_id, airline, " ".join(destination), ", ".join(flightNumbers), scheduledTime, estimatedTime,
scheduledDate, latestTime, status, statusColor)]
mycursor.executemany(sql, val)
mydb.commit()
print(mycursor.rowcount, "was inserted.")
print(airline, destination, flightNumbers, scheduledTime, "Estimated Time:" + " " + estimatedTime, "Scheduled Date:" + " " + scheduledDate, "Latest Time:" + " " + latestTime, "Status:" + " " +status, "Status Color:" + " " + statusColor)

Remove the trailing , in val and change val to be
val = [(flight_id, airline, destination, flightNumbers, scheduledTime, estimatedTime, scheduledDate, latestTime, status, statusColor)]
And don't forget to fix your sql to
sql = "INSERT INTO flightinfo (flight_id, airline, destination, flightNumbers, scheduledTime, estimatedTime, scheduledDate, latestTime, status, statusColor) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
UPDATE:
And to avoid getting
TypeError: Python 'list' cannot be converted to a MySQL type
You can convert the relevant fields to string for instance in your case you can do.
val = [(flight_id, airline, " ".join(destination), ", ".join(flightNumbers), scheduledTime, estimatedTime, scheduledDate, latestTime, status, statusColor)]

My Python program it's not writing sql database

This is my code, i'm getting data from a MCP3008 and I want to write that values into a SQL Database, but it's not working. When I execute the code it's okay, but when I open the SQL Database it's empty.
Python Program:
spi = spidev.SpiDev()
spi.open(0,0)
def ReadChannel(channel):
adc = spi.xfer2([1,(8+channel)<<4,0])
data = ((adc[1]&3) << 8) + adc[2]
return data
TIMES = 480
def AdcRead(adc_value = []):
time_start = time.time()
i = 0
while True:
time_current = time.time()
if time_current > time_start + i / float(TIMES):
print('{}: {}'.format(i, time_current))
data = ReadChannel(0)
adc_value.append(data)
i += 1
if i > 223:
max_value = max(adc_value)
break
print(adc_value)
return max_value
amp = AdcRead() * 0.8
amp = amp + 0.0
print("Binario: {}").format(amp)
output = 240*(amp/1024)*30
print("Potennia: {}").format(output)
amp_out = output/240
print ("Amperes: {}").format(amp_out)
output_h = output/3600
price = output_h * 0.15
ts = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
db = MySQLdb.connect("localhost","root","pass","auto_room_control")
cursor = db.cursor()
sql = "INSERT INTO auto_room_control VALUES ('%s', '%d', '%d', '%d', '%d' )", (ts, amp_out, output, output_h, price)
try:
cursor.execute(sql)
db.commit()
except:
db.rollback()
db.close()
And when I execute the MySql command:
mysql> USE auto_room_control;
mysql> SELECT * FROM power_consumption;
Empty set (0.00 sec)
I get that "Empty set(0.00 sec)". What am I doing wrong?

There are several issues.
Change
sql = "INSERT INTO auto_room_control VALUES ('%s', '%d', '%d', '%d', '%d' )", (ts, amp_out, output, output_h, price)
try:
cursor.execute(sql)
db.commit()
except:
db.rollback()
to
sql = "INSERT INTO auto_room_control VALUES (%s, %s, %s, %s, %s)"
sqldata = (ts, amp_out, output, output_h, price)
try:
cursor.execute(sql, sqldata)
db.commit()
except:
db.rollback()
because the %s are interpreted by the database driver and that is the cleanest way to prevent SQL injection.
Change this to
sql = "INSERT INTO auto_room_control VALUES (%s, %s, %s, %s, %s)"
sqldata = (ts, amp_out, output, output_h, price)
with db as cursor:
cursor.execute(sql, sqldata)
The with statement does the committing and rolling back on its own.

The problem is here:
sql = "INSERT INTO auto_room_control VALUES ('%s', '%d', '%d', '%d', '%d' )", (ts, amp_out, output, output_h, price)
You should write this:
sql = "INSERT INTO auto_room_control VALUES ('%s', %d, %d, %d, %d )" % (ts, amp_out, output, output_h, price)
In the your case, you just create couple of string and another couple, instead of formatting.

Pipeline Scrapy Python - Key Error: 'id'

I'm trying to get the scrapy pipeline working to funnel data into my db.
The problem I'm having is being unable to store a value for the primary field in my table that is "contentid.' I'm using the variable current_affiliate_item but not having any luck.
Thanks in advance for your help!
File "/Users/pipelines.py", line 46, in _conditional_insert
self.update_affiliate_item(tx, item, affiliate_item)
File "/Users/pipelines.py", line 109, in update_affiliate_item
current_affiliate_item['id'], #used to be this - current_affiliate_item['id']
exceptions.KeyError: 'id'
This is the code it is calling - beginning with line 46
ln 46 is - self.update_affiliate_item(tx, item, affiliate_item)
ln 109 is - current_affiliate_item['id']
I don't know what my problem is here, and have been stumped for days on it. I'm trying to call
if affiliate_item:
self.update_affiliate_item(tx, item, affiliate_item)
item_affiliate_id = affiliate_item['id']
item['affiliate_item_id'] = affiliate_item['id']
item['id'] = affiliate_item['item_id']
else:
item['id'] = self.get_item_id(tx, item)
item_affiliate_id = self.insert_affiliate_item(tx, item)
# item_affiliate_id = self.insert_or_update_affiliate(tx, item)
self.insert_or_update_photos(tx, item)
for price in item['prices']:
self.insert_or_update_price(tx, item, item_affiliate_id, price)
# log.msg("Item prices stored in db: %s" % item['name'], level=log.INFO)
def find_affiliate_item(self, tx, item): #changed selx to self
tx.execute("select * from z2ah5_jreviews_content where affiliate_item_id = %s and jr_retailer = %s", (item['affiliate_item_id'], item['retailer']))
result = tx.fetchone()
if result:
return result
else:
return False
def get_item_id(self, tx, item):
tx.execute("select * from z2ah5_jreviews_content where slug = %s ", (item['slug']))
result = tx.fetchone()
if result:
return result['contentid']
else:
tx.execute(\
"insert into z2ah5_jreviews_content (name, slug, date_created, date_modified) "
"values (%s, %s, %s, %s)",
(
item['name'],
item['slug'],
datetime.datetime.now(),
datetime.datetime.now(),
)
)
return tx.lastrowid
def insert_affiliate_item(self, tx, item):
tx.execute(\
"insert into z2ah5_jreviews_content (item_id, jr_retailer, affiliate_item_id, jr_description, date_created, date_modified) "
"values (%s, %s, %s, %s, %s, %s)",
(
item['id'],
item['retailer'],
item['affiliate_item_id'],
item['desc'],
datetime.datetime.now(),
datetime.datetime.now()
)
)
return tx.lastrowid
def update_affiliate_item(self, tx, item, current_affiliate_item): #used to be this - current_affiliate_item
tx.execute(\
"update z2ah5_jreviews_content set jr_description = %s, date_modified = %s where contentid = %s ",
(
item['desc'],
datetime.datetime.now(),
current_affiliate_item['id'], #used to be this - current_affiliate_item['id']
)
)

The error is due to the fact that the dict affiliate_item does not have an 'id' key when you call the update_affiliate_item function
instead of this
if affiliate_item:
try this:
if affiliate_item and affiliate_item.has_key('id'):