I'm trying to get the scrapy pipeline working to funnel data into my db.
The problem I'm having is being unable to store a value for the primary field in my table that is "contentid.' I'm using the variable current_affiliate_item but not having any luck.
Thanks in advance for your help!
File "/Users/pipelines.py", line 46, in _conditional_insert
self.update_affiliate_item(tx, item, affiliate_item)
File "/Users/pipelines.py", line 109, in update_affiliate_item
current_affiliate_item['id'], #used to be this - current_affiliate_item['id']
exceptions.KeyError: 'id'
This is the code it is calling - beginning with line 46
ln 46 is - self.update_affiliate_item(tx, item, affiliate_item)
ln 109 is - current_affiliate_item['id']
I don't know what my problem is here, and have been stumped for days on it. I'm trying to call
if affiliate_item:
self.update_affiliate_item(tx, item, affiliate_item)
item_affiliate_id = affiliate_item['id']
item['affiliate_item_id'] = affiliate_item['id']
item['id'] = affiliate_item['item_id']
else:
item['id'] = self.get_item_id(tx, item)
item_affiliate_id = self.insert_affiliate_item(tx, item)
# item_affiliate_id = self.insert_or_update_affiliate(tx, item)
self.insert_or_update_photos(tx, item)
for price in item['prices']:
self.insert_or_update_price(tx, item, item_affiliate_id, price)
# log.msg("Item prices stored in db: %s" % item['name'], level=log.INFO)
def find_affiliate_item(self, tx, item): #changed selx to self
tx.execute("select * from z2ah5_jreviews_content where affiliate_item_id = %s and jr_retailer = %s", (item['affiliate_item_id'], item['retailer']))
result = tx.fetchone()
if result:
return result
else:
return False
def get_item_id(self, tx, item):
tx.execute("select * from z2ah5_jreviews_content where slug = %s ", (item['slug']))
result = tx.fetchone()
if result:
return result['contentid']
else:
tx.execute(\
"insert into z2ah5_jreviews_content (name, slug, date_created, date_modified) "
"values (%s, %s, %s, %s)",
(
item['name'],
item['slug'],
datetime.datetime.now(),
datetime.datetime.now(),
)
)
return tx.lastrowid
def insert_affiliate_item(self, tx, item):
tx.execute(\
"insert into z2ah5_jreviews_content (item_id, jr_retailer, affiliate_item_id, jr_description, date_created, date_modified) "
"values (%s, %s, %s, %s, %s, %s)",
(
item['id'],
item['retailer'],
item['affiliate_item_id'],
item['desc'],
datetime.datetime.now(),
datetime.datetime.now()
)
)
return tx.lastrowid
def update_affiliate_item(self, tx, item, current_affiliate_item): #used to be this - current_affiliate_item
tx.execute(\
"update z2ah5_jreviews_content set jr_description = %s, date_modified = %s where contentid = %s ",
(
item['desc'],
datetime.datetime.now(),
current_affiliate_item['id'], #used to be this - current_affiliate_item['id']
)
)
The error is due to the fact that the dict affiliate_item does not have an 'id' key when you call the update_affiliate_item function
instead of this
if affiliate_item:
try this:
if affiliate_item and affiliate_item.has_key('id'):
Related
I'm a beginner and have just started exploring Tkinter and python programming. I came across a Tkinter program template which I would like to test some features on. Currently, this section allows user to input the ID values, and from the DB, the varchar is set to 45. I have a barcode reader in which when I scan produces a long output like "ID:ffffff100.....n SN:SK1980....n". Are there any way for me to only accept the values from "ffffff100....n" Only ? So that when I scan and ADD, it only inserts from "fff till n(excluding ID: and SN:SK19..n) ? What choices do I have?enter image description here I have attached some codes segment which I believe is the relevant segment and images for your reference. Thanks.
#Entry Type: Register New Asset
elif (choices == "Register New Asset"):
test1 = Check(temp_Entry.get(), gps_Entry.get(), current_Entry.get())
if (test1 == "TRUE"):
command = ("SELECT `Asset ID` FROM `Asset IDs` WHERE `Asset ID` = %s")
mycursor.execute(command, (ID_Box.get(), ))
results = mycursor.fetchall()
if len(results) == 0:
command = ("INSERT INTO `Asset IDs` (`Asset ID`) VALUES (%s)")
mycursor.execute(command, (ID_Box.get(), ))
connection.commit()
command = ("SELECT `Device EUI` FROM `Temp/Hum Sensors` WHERE `Device EUI` = %s")
mycursor.execute(command, (temp_Entry.get(), ))
results = mycursor.fetchall()
if len(results) == 0:
command = ("INSERT INTO `Temp/Hum Sensors` (`Device EUI`) VALUES (%s)")
mycursor.execute(command, (temp_Entry.get(), ))
connection.commit()
command = ("SELECT `Device EUI` FROM `GPS Sensors` WHERE `Device EUI` = %s")
mycursor.execute(command, (gps_Entry.get(), ))
results = mycursor.fetchall()
if len(results) == 0:
command = ("INSERT INTO `GPS Sensors` (`Device EUI`) VALUES (%s)")
mycursor.execute(command, (gps_Entry.get(), ))
connection.commit()
command = ("SELECT `Device EUI` FROM `Current Sensors` WHERE `Device EUI` = %s")
mycursor.execute(command, (current_Entry.get(), ))
results = mycursor.fetchall()
if len(results) == 0:
command = ("INSERT INTO `Current Sensors` (`Device EUI`) VALUES (%s)")
mycursor.execute(command, (current_Entry.get(), ))
connection.commit()
command = ("INSERT INTO `Inventory` (`Asset ID`, `Equipment Type`, `Temp/Hum EUI`, `GPS EUI`, `Current EUI`, `Status`, `Remarks`) VALUES (%s, %s, %s, %s, %s, %s, %s)")
val = (ID_Box.get(), asset_Box.get(), temp_Entry.get(), gps_Entry.get(), current_Entry.get(), status_Entry.get(), remarks_Entry.get())
mycursor.execute(command, val)
connection.commit()
tkinter.messagebox.showinfo("Register New Asset", "Successfully registered!")
clear_fields([asset_Box, ID_Box, temp_Entry, gps_Entry, current_Entry, name_Entry, company_Entry, email_Entry, contact_Entry, issue_Entry, return_Entry, status_Entry, remarks_Entry, issuehours, issueminutes, issuedays, returnhours, returnminutes, returndays])
entry_Box.configure(state="normal")
entry_Box.delete(0, "end")
entry_Box.insert(0, "Select the following")
entry_Box.configure(state="readonly")
entry_Box.focus()
addbutton.configure(state="disable")
else:
valueErrorMessage = "Invalid input for " + test1
tkinter.messagebox.showerror("Error", valueErrorMessage)
command = "SELECT `Asset ID` from `Inventory`"
mycursor.execute(command)
results = mycursor.fetchall()
enter image description here
enter image description here
I wrote this sql code in my database and it worked
INSERT INTO moni (sn, dgw, tgw, delay_gw_server) VALUES ('2020060002', '2020-07-05', '11:12:17', timediff(NOW(), (cast(concat(dgw, ' ', tgw) as datetime))))
can I use this code into python code?
date = ‘2020-07-07’
time = ‘17:17:17’
concat = ‘%s %s’ %(date, time)
dt = datetime.datetime.strptime(concat, ‘%Y-%m-%d %H:%M:%S’)
diff = (datetime.datetime.now() - dt)
mycursor = mydb.cursor()
sql = "INSERT INTO moni (sn, dgw, tgw, delay_gw_server) VALUES (%s, %s, %s, %s)"
val = (sn, date, time, diff)
mycursor.execute(sql, val)
mydb.commit()
or this code ?
date = ‘2020-07-07’
time = ‘17:17:17’
concat = ‘%s %s’ %(date, time)
dt = datetime.datetime.strptime(concat, ‘%Y-%m-%d %H:%M:%S’)
mycursor = mydb.cursor()
sql = "INSERT INTO moni (sn, dgw, tgw, delay_gw_server) VALUES (%s, %s, %s, %s)"
val = (sn, date, time, timediff(NOW(), dt))
mycursor.execute(sql, val)
mydb.commit()
I have tried several variation of this, and for that reason I am coming here for guidance. Where is this incorrect?
with connection.cursor() as cur:
sql = 'select * from table where var1 = %s, var2 = %s, var3 = %s, var4 = %s, var5 = %s'
cur.execute(sql, (var1val, var2val, var3val, var4val, var5val))
connection.commit()
l_fetch = cur.fetchall()
rc = int(l_fetch.rowcount)
print('rc len lerr_log: ' + rc)
if(rc > 0):
#result found
cur.fetchall() returns a list, it doesn't have a rowcount attribute.
The number of rows is cur.rowcount and len(l_fetch).
I am trying to do an insert from Python to MySQL and I am getting
mysql.connector.errors.ProgrammingError: Not enough parameters for the SQL statement;
I have had a bit of a look online and I know its got something to do with tuble but I can't work out what to change, I have looked around and at my code, there are 10 items in the responce_data, 10 in the SQL and I have 10 %s so I don't know where I am going wrong
import urllib.parse
import requests
import mysql.connector
mydb = mysql.connector.connect(
host="localhost",
user="root",
passwd="**",
database="flightdata"
)
mycursor = mydb.cursor()
main_api = 'https://www.sydneyairport.com.au/_a/flights/?query=&flightType=departure&terminalType=domestic&date=2019-11-10&sortColumn=scheduled_time&ascending=true&showAll=true'
address = 'lhr'
url = main_api + urllib.parse.urlencode({address: address})
response_data = requests.get(url).json()
for element in response_data['flightData']:
flight_id = element['id']
airline = element['airline']
destination = element['destinations']
flightNumbers = element['flightNumbers']
scheduledTime = element['scheduledTime']
estimatedTime = element['estimatedTime']
scheduledDate = element['scheduledDate']
latestTime = element['latestTime']
status = element['status']
statusColor = element['statusColor']
sql = "INSERT INTO flightinfo (id, airline, destinations, flightNumbers, scheduledTime, estimatedTime, scheduledDate, latestTime, status, statusColor) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
#sql = "INSERT INTO flightinfo (flight_id, airline, destination, flightNumbers, scheduledTime, estimatedTime, estimatedTime, scheduledDate, latestTime, status, statusColor ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s ,%s))"
val = [(flight_id, airline, " ".join(destination), ", ".join(flightNumbers), scheduledTime, estimatedTime,
scheduledDate, latestTime, status, statusColor)]
mycursor.executemany(sql, val)
mydb.commit()
print(mycursor.rowcount, "was inserted.")
print(airline, destination, flightNumbers, scheduledTime, "Estimated Time:" + " " + estimatedTime, "Scheduled Date:" + " " + scheduledDate, "Latest Time:" + " " + latestTime, "Status:" + " " +status, "Status Color:" + " " + statusColor)
Remove the trailing , in val and change val to be
val = [(flight_id, airline, destination, flightNumbers, scheduledTime, estimatedTime, scheduledDate, latestTime, status, statusColor)]
And don't forget to fix your sql to
sql = "INSERT INTO flightinfo (flight_id, airline, destination, flightNumbers, scheduledTime, estimatedTime, scheduledDate, latestTime, status, statusColor) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
UPDATE:
And to avoid getting
TypeError: Python 'list' cannot be converted to a MySQL type
You can convert the relevant fields to string for instance in your case you can do.
val = [(flight_id, airline, " ".join(destination), ", ".join(flightNumbers), scheduledTime, estimatedTime, scheduledDate, latestTime, status, statusColor)]
This is my code, i'm getting data from a MCP3008 and I want to write that values into a SQL Database, but it's not working. When I execute the code it's okay, but when I open the SQL Database it's empty.
Python Program:
spi = spidev.SpiDev()
spi.open(0,0)
def ReadChannel(channel):
adc = spi.xfer2([1,(8+channel)<<4,0])
data = ((adc[1]&3) << 8) + adc[2]
return data
TIMES = 480
def AdcRead(adc_value = []):
time_start = time.time()
i = 0
while True:
time_current = time.time()
if time_current > time_start + i / float(TIMES):
print('{}: {}'.format(i, time_current))
data = ReadChannel(0)
adc_value.append(data)
i += 1
if i > 223:
max_value = max(adc_value)
break
print(adc_value)
return max_value
amp = AdcRead() * 0.8
amp = amp + 0.0
print("Binario: {}").format(amp)
output = 240*(amp/1024)*30
print("Potennia: {}").format(output)
amp_out = output/240
print ("Amperes: {}").format(amp_out)
output_h = output/3600
price = output_h * 0.15
ts = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
db = MySQLdb.connect("localhost","root","pass","auto_room_control")
cursor = db.cursor()
sql = "INSERT INTO auto_room_control VALUES ('%s', '%d', '%d', '%d', '%d' )", (ts, amp_out, output, output_h, price)
try:
cursor.execute(sql)
db.commit()
except:
db.rollback()
db.close()
And when I execute the MySql command:
mysql> USE auto_room_control;
mysql> SELECT * FROM power_consumption;
Empty set (0.00 sec)
I get that "Empty set(0.00 sec)". What am I doing wrong?
There are several issues.
Change
sql = "INSERT INTO auto_room_control VALUES ('%s', '%d', '%d', '%d', '%d' )", (ts, amp_out, output, output_h, price)
try:
cursor.execute(sql)
db.commit()
except:
db.rollback()
to
sql = "INSERT INTO auto_room_control VALUES (%s, %s, %s, %s, %s)"
sqldata = (ts, amp_out, output, output_h, price)
try:
cursor.execute(sql, sqldata)
db.commit()
except:
db.rollback()
because the %s are interpreted by the database driver and that is the cleanest way to prevent SQL injection.
Change this to
sql = "INSERT INTO auto_room_control VALUES (%s, %s, %s, %s, %s)"
sqldata = (ts, amp_out, output, output_h, price)
with db as cursor:
cursor.execute(sql, sqldata)
The with statement does the committing and rolling back on its own.
The problem is here:
sql = "INSERT INTO auto_room_control VALUES ('%s', '%d', '%d', '%d', '%d' )", (ts, amp_out, output, output_h, price)
You should write this:
sql = "INSERT INTO auto_room_control VALUES ('%s', %d, %d, %d, %d )" % (ts, amp_out, output, output_h, price)
In the your case, you just create couple of string and another couple, instead of formatting.