Develop Reference

error occurred while inserting data mysql database

class MysqlPipeline(object):
def __init__(self):
**Connect the mysql**
self.conn = MySQLdb.connect('localhost','root','root','zhihu',
charset='utf8')
self.cursor = self.conn.cursor()
def process_item(self, item, spider):
**insert**
insert_sql = """
insert into
users_info(img_url,user_name,business,user_followingCount,
user_followerCount,idea_num,gender,favoriteCount,voteupCount,
followingColumnsCount,participatedLiveCount,followingFavlistsCount,
favoritedCount,uid,school_list,
job_list,place_list,major_list,company_list,url_token)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
"""
param= (item["img_url"],item["user_name"],item["business"],
item["user_followingCount"],item["user_followerCount"],
item["idea_num"],item["gender"],item["favoriteCount"],
item["voteupCount"],item["followingColumnsCount"],
item["participatedLiveCount"],
item["followingFavlistsCount"],
item["favoritedCount"],item["uid"],item["school_list"],
item["job_list"],item["place_list"],item["major_list"],
item["company_list"],item["url_token"]
)
self.cursor.execute(insert_sql,param)
Error
How should I solve this problem?
Traceback (most recent call last):
File "C:\Users\Administrator.JQ8B7UF6EZUHOOH\Envs\article_spider\lib\site-packages\twisted\internet\defer.py", line 653, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "D:/分布式爬虫相关测试/Zhihu\Zhihu\pipelines.py", line 38, in process_item
self.cursor.execute(insert_sql,param)
File "C:\Users\Administrator.JQ8B7UF6EZUHOOH\Envs\article_spider\lib\site-packages\MySQLdb\cursors.py", line 250, in execute
self.errorhandler(self, exc, value)
File "C:\Users\Administrator.JQ8B7UF6EZUHOOH\Envs\article_spider\lib\site-packages\MySQLdb\connections.py", line 50, in defaulterrorhandler
raise errorvalue
File "C:\Users\Administrator.JQ8B7UF6EZUHOOH\Envs\article_spider\lib\site-packages\MySQLdb\cursors.py", line 247, in execute
res = self._query(query)
File "C:\Users\Administrator.JQ8B7UF6EZUHOOH\Envs\article_spider\lib\site-packages\MySQLdb\cursors.py", line 411, in _query
rowcount = self._do_query(q)
File "C:\Users\Administrator.JQ8B7UF6EZUHOOH\Envs\article_spider\lib\site-packages\MySQLdb\cursors.py", line 374, in _do_query
db.query(q)
File "C:\Users\Administrator.JQ8B7UF6EZUHOOH\Envs\article_spider\lib\site-packages\MySQLdb\connections.py", line 277, in query
_mysql.connection.query(self, query)
_mysql_exceptions.ProgrammingError: (1064, "You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '),(),(),(),(),'qiu-shuo-47')' at line 4")

You are trying to insert 19 values in 20 columns:
print(len(param)) # 19
print(insert_sql.count('%s')) # 20

Failed to insert data into the table of mysql

When I try to insert the data into mysql,also create a table in the db "db_of_lj",some error occurs but I do not know the reason.
Also, I try to print(cursor.description) but the output is "None".
I hope you can help me,thanks!
import pymysql
def create_city_table(city):
#create table
global cursor
cursor.execute("drop table if exists %s" % (city))
create_table="create table %s(id int(10) not null primary key auto_increment," \
"total_price int(10)," \
"unitPrice int(10)," \
"room_desc varchar(20)," \
"construction_year int(10)," \
"area float," \
"location varchar(20));"
try:
cursor.execute(create_table % (city))
except :
print("Failed to create that table!")
return None
print("Successfully create table of '%s' in sql_db!" % city)
#connect to mysql
connect=pymysql.Connect(
host="127.0.0.1",
port=3306,
user="root",
password="1111",
db="db_of_lj"
)
#set a cursor
cursor = connect.cursor()
create_city_table("hangzhou")
insert2db = "insert into hangzhou(id,total_price,unitPrice,room_desc,construction_year,area,location)" \
" values(%d,%d,%d,%s,%d,%lf,%s)"
try:
print(cursor.description)
cursor.execute(insert2db % (1,2,3,"room_desc",2016,2.4,"xiasha"))
connect.commit()
except:
connect.rollback()
print("Failed to insert into database!")
exit(0)
print("Insert into the database successfully!")
Without try-catch the error return is:
Traceback (most recent call last):
File "D:/desktop/Computer/Python/web_scrapping/;project3_lianjia/exm.py", line 40, in <module>
cursor.execute(insert2db % (1,2,3,"room_desc",2016,2.4,"xiasha"))
File "C:\Users\94257\AppData\Local\Programs\Python\Python36-32\lib\site-packages\pymysql\cursors.py", line 165, in execute
result = self._query(query)
File "C:\Users\94257\AppData\Local\Programs\Python\Python36-32\lib\site-packages\pymysql\cursors.py", line 321, in _query
conn.query(q)
File "C:\Users\94257\AppData\Local\Programs\Python\Python36-32\lib\site-packages\pymysql\connections.py", line 860, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "C:\Users\94257\AppData\Local\Programs\Python\Python36-32\lib\site-packages\pymysql\connections.py", line 1061, in _read_query_result
result.read()
File "C:\Users\94257\AppData\Local\Programs\Python\Python36-32\lib\site-packages\pymysql\connections.py", line 1349, in read
first_packet = self.connection._read_packet()
File "C:\Users\94257\AppData\Local\Programs\Python\Python36-32\lib\site-packages\pymysql\connections.py", line 1018, in _read_packet
packet.check_error()
File "C:\Users\94257\AppData\Local\Programs\Python\Python36-32\lib\site-packages\pymysql\connections.py", line 384, in check_error
err.raise_mysql_exception(self._data)
File "C:\Users\94257\AppData\Local\Programs\Python\Python36-32\lib\site-packages\pymysql\err.py", line 107, in raise_mysql_exception
raise errorclass(errno, errval)
pymysql.err.InternalError: (1054, "Unknown column 'xiasha' in 'field list'")

Pandas.read_sql failing with DBAPIError 07002: COUNT field incorrect or syntax error

I am trying to execute a query and return the results to Excel. The query takes in a string of years as input parameters. I am calling it in Python like this:
def flatten(l):
for el in l:
try:
yield from flatten(el)
except TypeError:
yield el
my_list = (previous_year_1,previous_year,current_year)
sql = 'select year,sum(sales)/case when sum(t_count)=0 then 1 else sum(t_count) as tx_sales from t_sales where year in ({1})'+ 'group by year' + 'order by year'
sql = sql.format ('?',','.join('?' * len(my_list)))
params = tuple(flatten(member_list))
ind_data = pd.read_sql(sql,engine,params)
The query itself, after fixing the end clause, works perfectly when run through SSMS. Just not through the Python code. The error I'm getting is:
Traceback (most recent call last):
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 1139, in _execute_context
context)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\default.py", line 450, in do_execute
cursor.execute(statement, parameters)
pyodbc.Error: ('07002', '[07002] [Microsoft][SQL Server Native Client 11.0]COUNT field incorrect or syntax error (0) (SQLExecDirectW)')
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "c:\pbp_proj\pbp_proj.py", line 61, in pull_metrics
ind_data = pd.read_sql_query(sql, engine, params)
File "C:\Anaconda3\lib\site-packages\pandas\io\sql.py", line 411, in read_sql_query
parse_dates=parse_dates, chunksize=chunksize)
File "C:\Anaconda3\lib\site-packages\pandas\io\sql.py", line 1128, in read_query
result = self.execute(*args)
File "C:\Anaconda3\lib\site-packages\pandas\io\sql.py", line 1022, in execute
return self.engine.execute(*args, **kwargs)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 1989, in execute
return connection.execute(statement, *multiparams, **params)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 906, in execute
return self._execute_text(object, multiparams, params)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 1054, in _execute_text
statement, parameters
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 1146, in _execute_context
context)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 1341, in _handle_dbapi_exception
exc_info
File "C:\Anaconda3\lib\site-packages\sqlalchemy\util\compat.py", line 188, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=exc_value)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\util\compat.py", line 181, in reraise
raise value.with_traceback(tb)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\base.py", line 1139, in _execute_context
context)
File "C:\Anaconda3\lib\site-packages\sqlalchemy\engine\default.py", line 450, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.DBAPIError: (pyodbc.Error) ('07002', '[07002] [Microsoft][SQL Server Native Client 11.0]COUNT field incorrect or syntax error (0) (SQLExecDirectW)')
How can I resolve this error?

As #MYGz has already mentioned there is a missing space before order by.
Beside that there is a missing space before group by and the most important one - your CASE ... statement should be "closed" with END.
That said try the following SQL:
sql = 'select year,sum(sales)/(case when sum(t_count)=0 then 1 else sum(t_count) end)' \
+' as tx_sales from t_sales where year in ({1})'+' group by year order by year'
You can use your SQL pattern directly using .format() - there is no need to overwrite it:
params = tuple(flatten(member_list))
ind_data = pd.read_sql(sql.format('?',','.join('?' * len(params))), engine, params)

You have missed a space in your sql string between year and order by.
Try this:
sql = 'select year,sum(sales)/case when sum(t_count)=0 then 1 else sum(t_count) as tx_sales from t_sales where year in ({1}) '+ 'group by year ' + 'order by year '

Resolved this. A bit of hack, but works. I first changed it to using pyodbc instead of sqlalchemy.
so my query string became:
sql = 'select year,sum(sales)/case when sum(t_count)=0 then 1 else sum(t_count) end as tx_sales from t_sales where year in (?,?,?) '+ ' group by year' + ' order by year'
ind_data = pd.read_sql(sql, conn, params=member_list)
summary = ind_data.transpose()
I then had to add another AND clause with another parameter. For this I created:
cur_params = (member_list)
cur_params.append(var_premium)
then passsed cur_params to ind_data.
ind_data = pd.read_sql(sql, conn, params=cur_params)
both sets return data correctly now.
Thank you all for reading my post and for all the suggestions.

1064 error python to mysql

I have a database of names and prices that gets updated daily when I run it through a batch of code and update a second database all of the names until it gets to 'aapl' at which point it throws a 1064 error which looks like this
-----------------------------------
Traceback (most recent call last):
File "testrun.PY", line 45, in <module>
t.Push.find_all(conn, cursor)
File "c:\tradetools.py", line 198, in find_all
Push.find_streak(conn, cursor, name)
File "c:\tradetools.py", line 189, in find_strea
k
.format(c, name))
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\cursors.py", line 166, in execute
result = self._query(query)
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\cursors.py", line 322, in _query
conn.query(q)
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\connections.py", line 837, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\connections.py", line 1021, in _read_query_result
result.read()
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\connections.py", line 1304, in read
first_packet = self.connection._read_packet()
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\connections.py", line 983, in _read_packet
packet.check_error()
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\connections.py", line 395, in check_error
err.raise_mysql_exception(self._data)
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\err.py", line 102, in raise_mysql_exception
raise errorclass(errno, errval)
pymysql.err.ProgrammingError: (1064, "42000You have an error in your SQL syntax;
check the manual that corresponds to your MySQL server version for the right sy
ntax to use near 'AAPL''' at line 1")
c:\>
The code that its running through looks like this, why does the update add more commas to the name appl?
def find_streak(conn, cursor, name):
print(name)
cursor.execute("SELECT * FROM `trade_data`.`import_data`"
" WHERE name =%s;", name)
logs = cursor.fetchall()
cursor.execute("INSERT IGNORE INTO `trade_data`.`analysis`(`name`) "
"VALUES (%s) ON DUPLICATE KEY UPDATE "
"ndays=0;", name)
conn.commit()
logs = [list(x) for x in logs]
logs.sort()
....
cursor.execute ("UPDATE `trade_data`.`analysis` "
"SET `ndays` = {0} WHERE name='{1}'"
.format(c, name))
conn.commit()
its pulling from a table that looks like this
date|name|price|
and entering into a table that is
date|name|result

In SQLAlchemy, group_by on column property no longer works in 1.0.0b1

A change from 0.9.9 to 1.0.0b1 in the query functionality is causing me some heartburn. I have a group_by clause that uses a column_property.
In 0.9.9, the generated query reproduces the calculated value in GROUP BY by actually calculating the value again. In 1.0.0b1, the calculation is wrapped in an anon_1, and MSSQL won't let you group_by a named value for a calculated field.
Is there some way to revert to the old behavior without requiring a specific version?
The below code generates the following SQL in 0.9.9:
SELECT
count(cdr_extended.[UniqueID]) AS [CallCount],
sum(cdr_extended.[Duration]) AS [TotalSeconds],
ext_map.[FName] + ' ' + ext_map.[LName] AS anon_1
FROM cdr_extended, ext_map
WHERE
(ext_map.exten = cdr_extended.[Extension]
OR ext_map.prev_exten = cdr_extended.[Extension])
AND cdr_extended.[StartTime] > '2015-01-01'
AND cdr_extended.[Extension] IN ('8297')
GROUP BY ext_map.[FName] + ' ' + ext_map.[LName]
DESC
However, in 1.0.0, it produces this code:
SELECT
count(cdr_extended.[UniqueID]) AS [CallCount],
sum(cdr_extended.[Duration]) AS [TotalSeconds],
ext_map.[FName] + ' ' + ext_map.[LName] AS anon_1
FROM cdr_extended, ext_map
WHERE
(ext_map.exten = cdr_extended.[Extension]
OR ext_map.prev_exten = cdr_extended.[Extension])
AND cdr_extended.[StartTime] > '2015-01-01'
AND cdr_extended.[Extension] IN ('8297')
GROUP BY anon_1
DESC
Here's the model:
class EMap(Base):
FName = Column(String(length=45))
LName = Column(String(length=45))
AssociateName = column_property(FName + " " + LName)
The code in question:
DBSession.query(func.count(ExtendedCDR.UniqueID)
.label("CallCount"),func.sum(ExtendedCDR.Duration)
.label("TotalSeconds"))
.filter(or_(ExtensionMap.exten == ExtendedCDR.Extension,ExtensionMap.prev_exten == ExtendedCDR.Extension))
.filter(ExtendedCDR.StartTime>jan1)
.filter(ExtendedCDR.Extension.in_(extensions))
.group_by(ExtensionMap.AssociateName)
.order_by(func.count(ExtendedCDR.UniqueID).desc())
And finally, here's the actual stack trace when the group_by fails:
Traceback (most recent call last):
File "/usr/local/lib/python2.7/site-packages/pyramid_exclog-0.7-py2.7.egg/pyramid_exclog/__init__.py", line 111, in exclog_tween
return handler(request)
File "/usr/local/lib/python2.7/site-packages/pyramid-1.5.4-py2.7.egg/pyramid/router.py", line 163, in handle_request
response = view_callable(context, request)
File "/usr/local/lib/python2.7/site-packages/pyramid-1.5.4-py2.7.egg/pyramid/config/views.py", line 245, in _secured_view
return view(context, request)
File "/usr/local/lib/python2.7/site-packages/pyramid-1.5.4-py2.7.egg/pyramid/config/views.py", line 355, in rendered_view
result = view(context, request)
File "/usr/local/lib/python2.7/site-packages/pyramid-1.5.4-py2.7.egg/pyramid/config/views.py", line 501, in _requestonly_view
response = view(request)
File "/opt/cedar/cedar/views/ViewMyDashboard.py", line 51, in MyDashboardView
YearList = ObstinateDatabaseQueryAll(DBSession.query(func.count(ExtendedCDR.UniqueID).label("CallCount"),func.sum(ExtendedCDR.Duration).label("TotalSeconds"),ExtensionMap.AssociateName).filter(or_(ExtensionMap.exten == ExtendedCDR.Extension,ExtensionMap.prev_exten == ExtendedCDR.Extension)).filter(ExtendedCDR.StartTime>year_today).filter(ExtendedCDR.Extension.in_(extensions)).group_by(ExtensionMap.AssociateName).order_by(func.count(ExtendedCDR.UniqueID).desc()))
File "/opt/cedar/cedar/controllers/db.py", line 40, in ObstinateDatabaseQueryAll
ret=query.all()
File "build/bdist.linux-x86_64/egg/sqlalchemy/orm/query.py", line 2408, in all
return list(self)
File "build/bdist.linux-x86_64/egg/sqlalchemy/orm/query.py", line 2525, in __iter__
return self._execute_and_instances(context)
File "build/bdist.linux-x86_64/egg/sqlalchemy/orm/query.py", line 2540, in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
File "build/bdist.linux-x86_64/egg/sqlalchemy/engine/base.py", line 914, in execute
return meth(self, multiparams, params)
File "build/bdist.linux-x86_64/egg/sqlalchemy/sql/elements.py", line 323, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "build/bdist.linux-x86_64/egg/sqlalchemy/engine/base.py", line 1010, in _execute_clauseelement
compiled_sql, distilled_params
File "build/bdist.linux-x86_64/egg/sqlalchemy/engine/base.py", line 1146, in _execute_context
context)
File "build/bdist.linux-x86_64/egg/sqlalchemy/engine/base.py", line 1332, in _handle_dbapi_exception
exc_info
File "build/bdist.linux-x86_64/egg/sqlalchemy/util/compat.py", line 199, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb)
File "build/bdist.linux-x86_64/egg/sqlalchemy/engine/base.py", line 1139, in _execute_context
context)
File "build/bdist.linux-x86_64/egg/sqlalchemy/engine/default.py", line 442, in do_execute
cursor.execute(statement, parameters)
ProgrammingError: (pyodbc.ProgrammingError) ('42S22', "[42S22] [FreeTDS][SQL Server]Invalid column name 'anon_1'. (207) (SQLExecDirectW)") [SQL: 'SELECT count(cdr_extended.[UniqueID]) AS [CallCount], sum(cdr_extended.[Duration]) AS [TotalSeconds], ext_map.[FName] + ? + ext_map.[LName] AS anon_1 \nFROM cdr_extended, ext_map \nWHERE (ext_map.exten = cdr_extended.[Extension] OR ext_map.prev_exten = cdr_extended.[Extension]) AND cdr_extended.[StartTime] > ? AND cdr_extended.[Extension] IN (?) GROUP BY anon_1 ORDER BY count(cdr_extended.[UniqueID]) DESC'] [parameters: (' ', datetime.datetime(2015, 1, 1, 0, 0), '8297')]

This was identified as a bug by the developer and the fix for it will be published in the 1.0.0b4 milestone.