The problem with the formation of SQL queries. Python, pymysql - python

I want to change the bot database from SQLite to MySQL (pymysql) and ran into a problem.
I have a function for generating sql queries, but for some reason it stopped working:
def update_format_with_args(sql, parameters: dict):
values = ", ".join([
f"{item} = ?" for item in parameters
])
sql = sql.replace("XXX", values)
return sql, tuple(parameters.values())
def update_settingsx(**kwargs):
with connection.cursor() as db:
sql = f"UPDATE main_settings SET XXX "
sql, parameters = update_format_with_args(sql, kwargs)
db.execute(sql, parameters)
db.commit()
Error:
update_settingsx(profit_buy=now_unix)
File "/root/rent/bot_id/utils/db_api/sqlite.py", line 154, in update_settingsx
db.execute(sql, parameters)
File "/usr/local/lib/python3.7/dist-packages/pymysql/cursors.py", line 168, in execute
query = self.mogrify(query, args)
File "/usr/local/lib/python3.7/dist-packages/pymysql/cursors.py", line 147, in mogrify
query = query % self._escape_args(args, conn)
TypeError: not all arguments converted during string formatting


Not every database connector supports the ? substitution scheme. Pymysql uses %s instead.
Another alternative would be to use named substition:
def update_format_with_args(sql, parameters: dict):
values = ", ".join([
f"{item} = %({item})s" for item in parameters
])
sql = sql.replace("XXX", values)
return sql, parameters

Related

How do I view raw SQL of the query generated by asyncpg?

As it is said in asyncpg Usage, I can use $n pattern for the arguments and execute a query this way, for example:
result = await conn.fetchval("SELECT $1", 42)
In this case the raw SQL would be SELECT 42. How do I get this raw text with an asyncpg function before execution? I am asking this is because I want to log queries in my project before they are applied.
query_tpl = "SELECT $1"
values = (42,)
sql = what_is_here(query_tpl, *values) # <- ???
print(sql) # Must be "SELECT 42"
result = await conn.fetchval(query_tpl, *values)

Why is this query not working from python to SQL?

I am new to coding and databases, I can not get the query to work if I write it long hand but I have a lot to carry out and want it in a function but cannot get it to work, it returns a parameters error
import mysql.connector
def connection_check_1(query, value):
mydb = mysql.connector.connect(
host="******",
user="*****",
passwd="*****",
database="****"
)
mycursor = mydb.cursor()
mycursor.execute(query, (value))
myresult = mycursor.fetchall()
mydb.close()
return myresult
value = "sheep"
query = 'select inlicence from licence where animal = %s'
myresult = connection_check_1(query, value)
print(myresult)
Here is the SQL table I have
create table licence
(
animal varchar (20) primary key,
inlicence int (1)
);
This is the error I get
Traceback (most recent call last):
File "*******************", line 20, in
myresult = connection_check_1(query, value)
File "********************", line 13, in connection_check_1
mycursor.execute(query, (value))
File "********************************************88", line 246, in execute
prepared = self._cnx.prepare_for_mysql(params)
File "/home/kev/PycharmProjects/test bed/venv/lib/python3.5/site-packages/mysql/connector/connection_cext.py", line 535, in prepare_for_mysql
raise ValueError("Could not process parameters")
ValueError: Could not process parameters
I have tried changing the way the query is written, changing it to fetchall().

Wrapping a value with () doesn't turn it in to a tuple. You probably meant to add a comma there:
mycursor.execute(query, (value,))
# Creates a one-element tuple-^

web crawler not working properly

I have created a webcrawler in python 2.7 and i am using mysqldb to insert data into database.
I have executed each function as a different script for different webpages, but after i put them into a single file as functions, the program shows error;
(After entering seed page and depth)
Traceback (most recent call last):
File "C:\Users\Chetan\Desktop\webCrawler.py", line 207, in
mainFunc(depth,url)
File "C:\Users\Chetan\Desktop\webCrawler.py", line 194, in mainFunc
lst=perPage(url)
File "C:\Users\Chetan\Desktop\webCrawler.py", line 186, in perPage
filterContent(url,page)
File "C:\Users\Chetan\Desktop\webCrawler.py", line 149, in filterContent
cursor.execute(sql)
File "C:\Python27\lib\site-packages\MySQLdb\cursors.py", line 202, in execute
self.errorhandler(self, exc, value)
File "C:\Python27\lib\site-packages\MySQLdb\connections.py", line 36, in defaulterrorhandler
raise errorclass, errorvalue
ProgrammingError: (1064, 'You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near \'s and specials." />\n
I can't seem to find any problem. Here is the code;
def metaContent(page,url):#EXTRACTS META TAG CONTENT
lst=[]
while page.find("<meta")!=-1:
start_link=page.find("<meta")
page=page[start_link:]
start_link=page.find("content=")
start_quote=page.find('"',start_link)
end_quote=page.find('"',start_quote+1)
metaTag=page[start_quote+1:end_quote]
page=page[end_quote:]
lst.append(metaTag)
#ENTER DATA INTO DB
i,j=0,0
while i<len(lst):
sql = "INSERT INTO META(URL, \
KEYWORD) \
VALUES ('%s','%s')" % \
(url,lst[i])
cursor.execute(sql)
db.commit()
def filterContent(page,url):#FILTERS THE CONTENT OF THE REMAINING PORTION
phrase = ['to','a','an','the',"i'm",\
'for','from','that','their',\
'i','my','your','you','mine',\
'we','okay','yes','no','as',\
'if','but','why','can','now',\
'are','is','also']
#CALLS FUNC TO REMOVE HTML TAGS
page = strip_tags(page)
#CONVERT TO LOWERCASE
page = page.lower()
#REMOVES WHITESPACES
page = page.split()
page = " ".join(page)
#REMOVES IDENTICAL WORDS AND COMMON WORDS
page = set(page.split())
page.difference_update(phrase)
#CONVERTS FROM SET TO LIST
lst = list(page)
#ENTER DATA INTO DB
i,j=0,0
while i<len(lst):
sql = "INSERT INTO WORDS(URL, \
KEYWORD) \
VALUES ('%s','%s')" % \
(url,lst[i])
cursor.execute(sql)
db.commit()
#<6>
def perPage(url):#CALLS ALL THE FUNCTIONS
page=pageContent(url)
#REMOVES CONTENT BETWEEN SCRIPT TAGS
flg=0
while page.find("<script",flg)!=-1:
start=page.find("<script",flg)
end=page.find("</script>",flg)
end=end+9
i,k=0,end-start
page=list(page)
while i<k:
page.pop(start)
i=i+1
page=''.join(page)
flg=start
#REMOVES CONTENT BETWEEN STYLE TAGS
flg=0
while page.find("<script",flg)!=-1:
start=page.find("<style",flg)
end=page.find("</style>",flg)
end=end+9
i,k=0,end-start
page=list(page)
while i<k:
page.pop(start)
i=i+1
page=''.join(page)
flg=start
metaContent(url,page)
lst=linksExt(url,page)
filterContent(url,page)
return lst#CHECK WEATHER NEEDED OR NOT
#<7>
crawled=[]
def mainFunc(depth,url):#FOR THE DEPTH MANIPULATION
if (depth):
lst=perPage(url)
crawled.append(url)
i=0
if (depth-1):
while i<len(lst):
if url[i] not in crawled:
mainFunc(depth-1,url[i])
i+=1
#CALLING MAIN FUNCTION
mainFunc(depth,url)
Please mention any error, especially in depth manipulation function( mainFunc()). Anything regarding improving the crawler would be helpful.

It is definitely sql error, your quotes are not being escaped.
Instead of this
sql = "INSERT INTO META(URL, \
KEYWORD) \
VALUES ('%s','%s')" % \
(url,lst[i])
cursor.execute(sql)
and this
sql = "INSERT INTO WORDS(URL, \
KEYWORD) \
VALUES ('%s','%s')" % \
(url,lst[i])
cursor.execute(sql)
Try this
sql = "INSERT INTO WORDS(URL, \
KEYWORD) \
VALUES (%s, %s)"
cursor.execute(sql, (url, lst[i]))
and this
sql = "INSERT INTO META(URL, \
KEYWORD) \
VALUES (%s, %s)"
cursor.execute(sql, (url, lst[i]))
Also you are using while but not incrementing i, instead you can use this
for keyword in lst:
sql = "INSERT INTO META(URL, \
KEYWORD) \
VALUES (%s, %s)"
cursor.execute(sql, (url, keyword))

In the recursive call of mainFunc you are calling main function,
main(depth-1,url[i])
There is no main function in your code.
change it to,
mainFunc(depth-1,url[i])

python mysql connector query string syntax

I am new to python mysql connector and I am trying to get results for a query that will find the user_ids with p_id = 0 will go to product table find how many products are available in that city
import mysql.connector
con = mysql.connector.connect(user='user', password = 'pass', host = 'blah.com')
cursor1 = con.cursor(buffered = True)
query = ("SELECT l.user_id, count(u.prod_id)"
"FROM db1.users as l INNER JOIN db2.product as u "
"ON l.u_city = u.p_city"
"WHERE l.p_id =0 GROUP BY l.user_id limit 10;" )
cursor1.execute(query)
the query is getting executed mysql but from python mysql connector i am getting the following error
C:\Python27\python.exe C:/Python27/Lib/site-packages/mysql/connector/update_campus_user_profile_suset.py
Traceback (most recent call last):
File "C:/Python27/Lib/site-packages/mysql/connector/update_campus_user_profile_suset.py", line 12, in <module>
cursor1.execute(camp_zip)
File "C:\Python27\lib\site-packages\mysql\connector\cursor.py", line 491, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "C:\Python27\lib\site-packages\mysql\connector\connection.py", line 683, in cmd_query
statement))
File "C:\Python27\lib\site-packages\mysql\connector\connection.py", line 601, in _handle_result
raise errors.get_exception(packet)
mysql.connector.errors.ProgrammingError: 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'l.campus_id <2 GROUP BY l.user_id' at line 1
Process finished with exit code 1

query ='''SELECT l.user_id, count(u.prod_id) FROM db1.users as l INNER JOIN db2.product as u ON l.u_city = u.p_city WHERE l.p_id =0 GROUP BY l.user_id limit 10'''
always it is better to intialize ur query to a variable with ('''stmt''') and execute
there is no need of using ;

It looks like you're just missing some spaces at the end of each line. Try:
query = ("SELECT l.user_id, count(u.prod_id) "
"FROM db1.users as l INNER JOIN db2.product as u "
"ON l.u_city = u.p_city "
"WHERE l.p_id = 0 GROUP BY l.user_id limit 10;" )
Sometimes it's more natural to use a multi-line string:
query = ("""
SELECT l.user_id, count(u.prod_id)
FROM db1.users as l INNER JOIN db2.product as u
ON l.u_city = u.p_city
WHERE l.p_id = 0 GROUP BY l.user_id limit 10
""")

like in mysql connector/python

I am using mysql-connector/python for the first time and I am facing an error.
I want to fetch the email addresses with net from a mysql table called blah.
cursor = cnx.cursor()
p = "net"
query = ("SELECT * FROM blah WHERE email LIKE %s limit 10", ("%" + p + "%",))
cursor.execute(query,(p,))
for row in cursor:
print row
cursor.close()
cnx.close()
and I am getting this error
Traceback (most recent call last):
File "<input>", line 9, in <module>
File "C:\Python27\lib\site-packages\mysql\connector\cursor.py", line 491, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "C:\Python27\lib\site-packages\mysql\connector\connection.py", line 683, in cmd_query
statement))
File "C:\Python27\lib\site-packages\mysql\connector\connection.py", line 601, in _handle_result
raise errors.get_exception(packet)
ProgrammingError: 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '"SELECT * FROM blah WHERE email LIKE %s limit 10", ("%" + p +"%")' at line 1
Can anyone suggest why is the code not working? It is working on sql -
select * from blah where email LIKE "%net" limit 10

Try this:
query = "SELECT * FROM blah WHERE email LIKE CONCAT('%', %s, '%') limit 10"
cursor.execute(query,(p,))
You need to leave %s in the string so that MySQL Connector can substitute the execute parameter for it. You were setting query to a list, not a string.

the first argument to cursor.execute should be a string it looks like you have munged assigning the string and calling the function. Probably should look something like:
query = "SELECT * FROM blah WHERE email LIKE %s limit 10"
cursor.execute(query,("%" + p + "%",))

Categories

Resources