Develop Reference

sqlite3 python generate tables

I am trying to generate tables based off the paramaters of a function. To generate the table I thought of passing a string of what I wanted to be executed as the following function:
conn = sqlite3.connect('tutorial.db')
c = conn.cursor()
def create_table(name, unix, datestamp, keyword, value):
command = "CREATE TABLE IF NOT EXISTS " + name + "(" + unix + " REAL, " + datestamp + " TEXT, " + keyword + " TEXT, " + value + " REAL)"
c.execute('CREATE TABLE' + command)
However when I run the command:
create_table('new','boy','girl','joy','joe')
I get error:
Traceback (most recent call last):
File "C:\Users\David\Documents\learn_sql_\learn_sql.py", line 22, in
create_table('new','boy','girl','joy','joe')
File "C:\Users\David\Documents\learn_sql_\learn_sql.py", line 12, in create_table
c.execute('CREATE TABLE' + command)
sqlite3.OperationalError: near "TABLECREATE": syntax error
Thoughts?
Thanks

try with this :
import sqlite3
conn = sqlite3.connect('tutorial.db')
c = conn.cursor()
def create_table(name, unix, datestamp, keyword, value):
#c.execute('CREATE TABLE' + command)
cmd = "CREATE TABLE IF NOT EXISTS %s(%s real,%s text,%s text,%s real)" % (name,unix,datestamp,keyword,value)
print(cmd)
c.execute(cmd)
create_table('new','boy','girl','joy','joe')

Insert into table from dynamic lists

I'm using dynamic method to create table and then insert values into it. There are two lists that dynamically filled from different files and these lists could be increased or decreased in count as well as the values are changing every time. My problem here I managed to create the table with its fields dynamically but I still have some issues inserting values into it. here is my code:
f = open(filepath,"r")
pluginoutput= f.read()
pluginoptojson = json.loads(pluginoutput)
columnsnames = (pluginoptojson["columns"])
countcolumns = len(pluginoptojson["columns"])
count = 0
lst = []
for name in columnsnames:
if count < countcolumns:
lst.append(str(name))
count +=1
lst.append("caseid")
table_name = "test2"
createsqltable = """CREATE TABLE IF NOT EXISTS """ + table_name + " (" + " VARCHAR(50),".join(lst) + " VARCHAR(50))"
print createsqltable
c.execute(createsqltable)
conn.commit()
#c.close()
#conn.close()
#gc.collect()
rowsvalue = (pluginoptojson["rows"])
for row in rowsvalue:
lst2 =[]
rowelementscount = len(row)
for value in row:
lst2.append(str(value))
lst2.append("test")
insert_intotable = "INSERT INTO " + table_name + "(" + ",".join(lst) + ") VALUES (" + ",".join(lst2) +")"
print insert_intotable
c.execute(insert_intotable)
conn.commit()
c.close()
conn.close()
gc.collect()
the file content as an example is :
{"lines": [[2, 12121, 33, 44, "ff"], [2, 786, 33, 66, "ww"]], "columns": ["id", "testnumber", "size", "gnumber", "filenname"]}
the print out of interesting values int table is :
INSERT INTO test8(id,testnumber,size,gnumber,filenname) VALUES (2,786,33, 66,ww,test)
the error I'm getting is:
Traceback (most recent call last):
File "pp.py", line 50, in
c.execute(insert_intotable)
File "/usr/lib/python2.7/dist-packages/MySQLdb/cursors.py", line 174, in execute
self.errorhandler(self, exc, value)
File "/usr/lib/python2.7/dist-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler
raise errorclass, errorvalue
_mysql_exceptions.ProgrammingError: (1064, "You have an error in your SQL syntax; check the manual that corresponds to your MySQL
server version for the right syntax to use near 'ww,test)' at line 1")
Any help will be appreciated :)

web crawler not working properly

I have created a webcrawler in python 2.7 and i am using mysqldb to insert data into database.
I have executed each function as a different script for different webpages, but after i put them into a single file as functions, the program shows error;
(After entering seed page and depth)
Traceback (most recent call last):
File "C:\Users\Chetan\Desktop\webCrawler.py", line 207, in
mainFunc(depth,url)
File "C:\Users\Chetan\Desktop\webCrawler.py", line 194, in mainFunc
lst=perPage(url)
File "C:\Users\Chetan\Desktop\webCrawler.py", line 186, in perPage
filterContent(url,page)
File "C:\Users\Chetan\Desktop\webCrawler.py", line 149, in filterContent
cursor.execute(sql)
File "C:\Python27\lib\site-packages\MySQLdb\cursors.py", line 202, in execute
self.errorhandler(self, exc, value)
File "C:\Python27\lib\site-packages\MySQLdb\connections.py", line 36, in defaulterrorhandler
raise errorclass, errorvalue
ProgrammingError: (1064, 'You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near \'s and specials." />\n
I can't seem to find any problem. Here is the code;
def metaContent(page,url):#EXTRACTS META TAG CONTENT
lst=[]
while page.find("<meta")!=-1:
start_link=page.find("<meta")
page=page[start_link:]
start_link=page.find("content=")
start_quote=page.find('"',start_link)
end_quote=page.find('"',start_quote+1)
metaTag=page[start_quote+1:end_quote]
page=page[end_quote:]
lst.append(metaTag)
#ENTER DATA INTO DB
i,j=0,0
while i<len(lst):
sql = "INSERT INTO META(URL, \
KEYWORD) \
VALUES ('%s','%s')" % \
(url,lst[i])
cursor.execute(sql)
db.commit()
def filterContent(page,url):#FILTERS THE CONTENT OF THE REMAINING PORTION
phrase = ['to','a','an','the',"i'm",\
'for','from','that','their',\
'i','my','your','you','mine',\
'we','okay','yes','no','as',\
'if','but','why','can','now',\
'are','is','also']
#CALLS FUNC TO REMOVE HTML TAGS
page = strip_tags(page)
#CONVERT TO LOWERCASE
page = page.lower()
#REMOVES WHITESPACES
page = page.split()
page = " ".join(page)
#REMOVES IDENTICAL WORDS AND COMMON WORDS
page = set(page.split())
page.difference_update(phrase)
#CONVERTS FROM SET TO LIST
lst = list(page)
#ENTER DATA INTO DB
i,j=0,0
while i<len(lst):
sql = "INSERT INTO WORDS(URL, \
KEYWORD) \
VALUES ('%s','%s')" % \
(url,lst[i])
cursor.execute(sql)
db.commit()
#<6>
def perPage(url):#CALLS ALL THE FUNCTIONS
page=pageContent(url)
#REMOVES CONTENT BETWEEN SCRIPT TAGS
flg=0
while page.find("<script",flg)!=-1:
start=page.find("<script",flg)
end=page.find("</script>",flg)
end=end+9
i,k=0,end-start
page=list(page)
while i<k:
page.pop(start)
i=i+1
page=''.join(page)
flg=start
#REMOVES CONTENT BETWEEN STYLE TAGS
flg=0
while page.find("<script",flg)!=-1:
start=page.find("<style",flg)
end=page.find("</style>",flg)
end=end+9
i,k=0,end-start
page=list(page)
while i<k:
page.pop(start)
i=i+1
page=''.join(page)
flg=start
metaContent(url,page)
lst=linksExt(url,page)
filterContent(url,page)
return lst#CHECK WEATHER NEEDED OR NOT
#<7>
crawled=[]
def mainFunc(depth,url):#FOR THE DEPTH MANIPULATION
if (depth):
lst=perPage(url)
crawled.append(url)
i=0
if (depth-1):
while i<len(lst):
if url[i] not in crawled:
mainFunc(depth-1,url[i])
i+=1
#CALLING MAIN FUNCTION
mainFunc(depth,url)
Please mention any error, especially in depth manipulation function( mainFunc()). Anything regarding improving the crawler would be helpful.

It is definitely sql error, your quotes are not being escaped.
Instead of this
sql = "INSERT INTO META(URL, \
KEYWORD) \
VALUES ('%s','%s')" % \
(url,lst[i])
cursor.execute(sql)
and this
sql = "INSERT INTO WORDS(URL, \
KEYWORD) \
VALUES ('%s','%s')" % \
(url,lst[i])
cursor.execute(sql)
Try this
sql = "INSERT INTO WORDS(URL, \
KEYWORD) \
VALUES (%s, %s)"
cursor.execute(sql, (url, lst[i]))
and this
sql = "INSERT INTO META(URL, \
KEYWORD) \
VALUES (%s, %s)"
cursor.execute(sql, (url, lst[i]))
Also you are using while but not incrementing i, instead you can use this
for keyword in lst:
sql = "INSERT INTO META(URL, \
KEYWORD) \
VALUES (%s, %s)"
cursor.execute(sql, (url, keyword))

In the recursive call of mainFunc you are calling main function,
main(depth-1,url[i])
There is no main function in your code.
change it to,
mainFunc(depth-1,url[i])

Creating a MySQL table's columns with a List

New to SO and fairly new to coding, so doing my best to follow the appropriate protocols.
In my python script, I'm creating a new table and populating column names from a list, named 'dups'.
dups = ['Id', 'Name', 'Price', 'Rating']
I'm inputting this list as columns for the new table, called "SuperTable", via a for loop. See code below:
with new_db:
cur = new_db.cursor()
cur.execute("DROP TABLE IF EXISTS SuperTable")
for i in dups:
if i == dups[0]:
new_col = i.replace("'","")
cur.execute("CREATE TABLE SuperTable(%s)" % (new_col))
else:
cur.execute("ALTER TABLE SuperTable ADD COLUMN %s" % i)
I've looked around a lot and can't seem to identify what I'm doing wrong. This approach worked with Sqlite but I keep getting this same error for MySQLdb:
Traceback (most recent call last):
File "MySQL_SuperTable.py", line 125, in <module>
cur.execute("CREATE TABLE Super(%s)" % (new_col))
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/site-packages/MySQLdb/cursors.py", line 174, in execute
self.errorhandler(self, exc, value)
File "/opt/local/Library/Frameworks/Python.framework/Versions/2.6/lib/python2.6/site-packages/MySQLdb/connections.py", line 36, in defaulterrorhandler
raise errorclass, errorvalue
_mysql_exceptions.ProgrammingError: (1064, "You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near ')' at line 1")

Thanks to eggyal! He pointed out that MySQL columns require a datatype. This is what the code looks like now (I created a list of tuples to input the datatypes + column names via a for loop):
with new_db:
cur = new_db.cursor()
cur.execute("DROP TABLE IF EXISTS SuperTable")
for i in col_namestypes:
if i == col_namestypes[0]:
cur.execute("CREATE TABLE SuperTable(%s %s)" % (i))
else:
cur.execute("ALTER TABLE SuperTable ADD COLUMN %s %s" % i)
for i in new_table:
count = len(i)
question_marks = []
while a < count:
question_marks.append('%s')
a += 1
quests = ','.join(question_marks)
cur.executemany("INSERT INTO SuperTable VALUES(%s)" % quests, new_table)

Python MySQLdb error - what is causing this

I am wondering why I am receiving this error:
cmd = "INSERT INTO resulttest (category, value, timestamp) VALUES (" + key + ", " + str(value) + ", " + str(timestamp) + ")"
c.execute(cmd)
db.commit()
INSERT INTO resulttest (category, value, timestamp) VALUES (composed, 2, 1343186948.8)
Traceback (most recent call last):
File "C:/Behavioral Technology/Google Drive/twitterthingv5test.py", line 94, in <module>
moodParser()
File "C:/Behavioral Technology/Google Drive/twitterthingv5test.py", line 92, in moodParser
query()
File "C:/Behavioral Technology/Google Drive/twitterthingv5test.py", line 37, in query
main(columns)
File "C:/Behavioral Technology/Google Drive/twitterthingv5test.py", line 81, in main
c.execute(cmd)
File "C:\Python27\lib\site-packages\MySQLdb\cursors.py", line 174, in execute
self.errorhandler(self, exc, value)
File "C:\Python27\lib\site-packages\MySQLdb\connections.py", line 36, in defaulterrorhandler
raise errorclass, errorvalue
I believe it has something to do with how I am passing my values to the SQL command.

Your code to create the query isn't attempting to quote the string values:
cmd = "INSERT INTO resulttest (category, value, timestamp) VALUES (" + key + ", " + str(value) + ", " + str(timestamp) + ")"
Look at the SQL statement you printed:
INSERT INTO resulttest (category, value, timestamp) VALUES (composed, 2, 1343186948.8)
Shouldn't "category" be quoted?
You shouldn't be composing a SQL statement with string operations in the first place. This is how SQL injection vulnerabilities happen. Instead you should use placeholders and let the MySQL library deal with them:
c.execute(
"INSERT INTO resulttest (category, value, timestamp) VALUES (?, ?, ?)",
(key, value, timestamp)
)

In case Ned Batchelder's suggestion doesn't work, here's an alternative:
sql = "INSERT into resulttest (category, value, timestamp) VALUES (%s, %s, %s)"
c.execute(sql % (key, value, timestamp))
I faced a problem where my SQL command wasn't being executed, and following this kind of a syntax made it work.
Although you must be careful, because using % instead of , opens up the possibility for SQL injections, but this was the only syntax that worked for me. Might be a Python-MySQL version and compatibility problem.
A better idea would be to install compatible versions and follow the correct syntax.