Develop Reference

unknown column in fireld list- mysql python connector-insertion

c.execute('show tables')
for i in c:
print(i)
t=input('enter exact table name')
n=0
c.execute('show columns from {}'.format(t))
records=c.fetchall()
print(records)
for i in records:
n=n+1
a=1
data=''
while a!=0:
for i in range(n):
v=input("enter data")
if i==n-1:
data=data+v
else:
data=data+v+','
print(data)
print('insert into {} values{}'.format(t,data))
c.execute('insert into {} values({})'.format(t,data))
in this code i am getting the following error:
Traceback (most recent call last):
File "C:\Users\subra\OneDrive\Desktop\netflix.py", line 62, in <module>
create_input()
File "C:\Users\subra\OneDrive\Desktop\netflix.py", line 51, in create_input
c.execute('insert into {} values{}'.format(t,data))
File "C:\Users\subra\AppData\Local\Programs\Python\Python38\lib\site-packages\mysql\connector\cursor.py", line 569, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "C:\Users\subra\AppData\Local\Programs\Python\Python38\lib\site-packages\mysql\connector\connection.py", line 599, in cmd_query
result = self._handle_result(self._send_cmd(ServerCmd.QUERY, query))
File "C:\Users\subra\AppData\Local\Programs\Python\Python38\lib\site-packages\mysql\connector\connection.py", line 487, in _handle_result
raise errors.get_exception(packet)
mysql.connector.errors.ProgrammingError: 1054 (42S22): Unknown column 'subru' in 'field list'
can anyone help me out on how to avoid this error
i have also made sure that there is only 1 table and that is the one i am inserting
this is the output that i get including the input vales:
>>>('hellopython',)
>>>enter exact table namehellopython
>>>[('student', b'varchar(30)', 'YES', '', None, ''), ('rollno', b'varchar(30)', 'YES', '', None,'')]
>>>enter datasubru
>>>enter data29
>>>(subru,29)
>>>insert into hellopython values(subru,29)
***Traceback (most recent call last):
File "C:\Users\subra\OneDrive\Desktop\netflix.py", line 62, in <module>
create_input()
File "C:\Users\subra\OneDrive\Desktop\netflix.py", line 51, in create_input
c.execute('insert into {} values{}'.format(t,data))
File "C:\Users\subra\AppData\Local\Programs\Python\Python38\lib\site-packages\mysql\connector\cursor.py", line 569, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "C:\Users\subra\AppData\Local\Programs\Python\Python38\lib\site-packages\mysql\connector\connection.py", line 599, in cmd_query
result = self._handle_result(self._send_cmd(ServerCmd.QUERY, query))
File "C:\Users\subra\AppData\Local\Programs\Python\Python38\lib\site-packages\mysql\connector\connection.py", line 487, in _handle_result
raise errors.get_exception(packet)
mysql.connector.errors.ProgrammingError: 1054 (42S22): Unknown column 'subru' in 'field list'***

You are missing quotes around the data.
So MySQL thinks that subru is not a name. instead he thninks it is a column name
if i==n-1:
data=data+"'"+v+ "'"
else:
data=data+"'"+v+ "'"+','
print(data)
print('insert into {} values{}'.format(t,data))
c.execute('insert into {} values({})'.format(t,data))

error occurred while inserting data mysql database

class MysqlPipeline(object):
def __init__(self):
**Connect the mysql**
self.conn = MySQLdb.connect('localhost','root','root','zhihu',
charset='utf8')
self.cursor = self.conn.cursor()
def process_item(self, item, spider):
**insert**
insert_sql = """
insert into
users_info(img_url,user_name,business,user_followingCount,
user_followerCount,idea_num,gender,favoriteCount,voteupCount,
followingColumnsCount,participatedLiveCount,followingFavlistsCount,
favoritedCount,uid,school_list,
job_list,place_list,major_list,company_list,url_token)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
"""
param= (item["img_url"],item["user_name"],item["business"],
item["user_followingCount"],item["user_followerCount"],
item["idea_num"],item["gender"],item["favoriteCount"],
item["voteupCount"],item["followingColumnsCount"],
item["participatedLiveCount"],
item["followingFavlistsCount"],
item["favoritedCount"],item["uid"],item["school_list"],
item["job_list"],item["place_list"],item["major_list"],
item["company_list"],item["url_token"]
)
self.cursor.execute(insert_sql,param)
Error
How should I solve this problem?
Traceback (most recent call last):
File "C:\Users\Administrator.JQ8B7UF6EZUHOOH\Envs\article_spider\lib\site-packages\twisted\internet\defer.py", line 653, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "D:/分布式爬虫相关测试/Zhihu\Zhihu\pipelines.py", line 38, in process_item
self.cursor.execute(insert_sql,param)
File "C:\Users\Administrator.JQ8B7UF6EZUHOOH\Envs\article_spider\lib\site-packages\MySQLdb\cursors.py", line 250, in execute
self.errorhandler(self, exc, value)
File "C:\Users\Administrator.JQ8B7UF6EZUHOOH\Envs\article_spider\lib\site-packages\MySQLdb\connections.py", line 50, in defaulterrorhandler
raise errorvalue
File "C:\Users\Administrator.JQ8B7UF6EZUHOOH\Envs\article_spider\lib\site-packages\MySQLdb\cursors.py", line 247, in execute
res = self._query(query)
File "C:\Users\Administrator.JQ8B7UF6EZUHOOH\Envs\article_spider\lib\site-packages\MySQLdb\cursors.py", line 411, in _query
rowcount = self._do_query(q)
File "C:\Users\Administrator.JQ8B7UF6EZUHOOH\Envs\article_spider\lib\site-packages\MySQLdb\cursors.py", line 374, in _do_query
db.query(q)
File "C:\Users\Administrator.JQ8B7UF6EZUHOOH\Envs\article_spider\lib\site-packages\MySQLdb\connections.py", line 277, in query
_mysql.connection.query(self, query)
_mysql_exceptions.ProgrammingError: (1064, "You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '),(),(),(),(),'qiu-shuo-47')' at line 4")

You are trying to insert 19 values in 20 columns:
print(len(param)) # 19
print(insert_sql.count('%s')) # 20

How do I reformat this json for database import?

I have these "json" files that I like to insert into my mongodb database.
An example of one is:
http://s.live.ksmobile.net/cheetahlive/de/ff/15201023827214369775/15201023827214369775.json
The problem is, that it is formated like this:
{ "channelType":"TEMPGROUP", ... } # line 1
{ "channelType":"TEMPGROUP", ... } # line 2
So instead of inserting it as 1 document in the DB, it insert every single line as 1 entry. That ends up with what should be 3 documents from 3 "json" files in the database become 1189 documents in the database instead.
How can I insert the whole content of the ".json" into one document?
My code is:
replay_url = "http://live.ksmobile.net/live/getreplayvideos?"
userid = 969730808384462848
url2 = replay_url + urllib.parse.urlencode({'userid': userid}) + '&page_size=1000'
raw_replay_data = requests.get(url2).json()
for i in raw_replay_data['data']['video_info']:
url3 = i['msgfile']
raw_message_data = urllib.request.urlopen(url3)
for line in raw_message_data:
json_data = json.loads(line)
messages.insert_one(json_data)
print(json_data)
Update to give more information to answer
messages.insert(json_data) gives this error:
Traceback (most recent call last):
File "/media/anon/06bcf743-8b4d-409f-addc-520fc4e19299/PycharmProjects/LiveMe/venv1/lib/python3.6/site-packages/pymongo/collection.py", line 633, in _insert
blk.execute(concern, session=session)
File "/media/anon/06bcf743-8b4d-409f-addc-520fc4e19299/PycharmProjects/LiveMe/venv1/lib/python3.6/site-packages/pymongo/bulk.py", line 432, in execute
return self.execute_command(generator, write_concern, session)
File "/media/anon/06bcf743-8b4d-409f-addc-520fc4e19299/PycharmProjects/LiveMe/venv1/lib/python3.6/site-packages/pymongo/bulk.py", line 329, in execute_command
raise BulkWriteError(full_result)
pymongo.errors.BulkWriteError: batch op errors occurred
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/media/anon/06bcf743-8b4d-409f-addc-520fc4e19299/PycharmProjects/LiveMe/import_messages_dev.py", line 43, in <module>
messages.insert(json_data)
File "/media/anon/06bcf743-8b4d-409f-addc-520fc4e19299/PycharmProjects/LiveMe/venv1/lib/python3.6/site-packages/pymongo/collection.py", line 2941, in insert
check_keys, manipulate, write_concern)
File "/media/anon/06bcf743-8b4d-409f-addc-520fc4e19299/PycharmProjects/LiveMe/venv1/lib/python3.6/site-packages/pymongo/collection.py", line 635, in _insert
_raise_last_error(bwe.details)
File "/media/anon/06bcf743-8b4d-409f-addc-520fc4e19299/PycharmProjects/LiveMe/venv1/lib/python3.6/site-packages/pymongo/helpers.py", line 220, in _raise_last_error
_raise_last_write_error(write_errors)
File "/media/anon/06bcf743-8b4d-409f-addc-520fc4e19299/PycharmProjects/LiveMe/venv1/lib/python3.6/site-packages/pymongo/helpers.py", line 188, in _raise_last_write_error
raise DuplicateKeyError(error.get("errmsg"), 11000, error)
pymongo.errors.DuplicateKeyError: E11000 duplicate key error index: liveme.messages.$_id_ dup key: { : ObjectId('5aa2fc6f5d60126499060949') }
messages.insert_one(json_data) gives me this error:
Traceback (most recent call last):
File "/media/anon/06bcf743-8b4d-409f-addc-520fc4e19299/PycharmProjects/LiveMe/import_messages_dev.py", line 43, in <module>
messages.insert_one(json_data)
File "/media/anon/06bcf743-8b4d-409f-addc-520fc4e19299/PycharmProjects/LiveMe/venv1/lib/python3.6/site-packages/pymongo/collection.py", line 676, in insert_one
common.validate_is_document_type("document", document)
File "/media/anon/06bcf743-8b4d-409f-addc-520fc4e19299/PycharmProjects/LiveMe/venv1/lib/python3.6/site-packages/pymongo/common.py", line 434, in validate_is_document_type
"collections.MutableMapping" % (option,))
TypeError: document must be an instance of dict, bson.son.SON, bson.raw_bson.RawBSONDocument, or a type that inherits from collections.MutableMapping
messages.insert_many(json_data) gives me this error:
Traceback (most recent call last):
File "/media/anon/06bcf743-8b4d-409f-addc-520fc4e19299/PycharmProjects/LiveMe/import_messages_dev.py", line 43, in <module>
messages.insert_many(json_data)
File "/media/anon/06bcf743-8b4d-409f-addc-520fc4e19299/PycharmProjects/LiveMe/venv1/lib/python3.6/site-packages/pymongo/collection.py", line 742, in insert_many
blk.execute(self.write_concern.document, session=session)
File "/media/anon/06bcf743-8b4d-409f-addc-520fc4e19299/PycharmProjects/LiveMe/venv1/lib/python3.6/site-packages/pymongo/bulk.py", line 432, in execute
return self.execute_command(generator, write_concern, session)
File "/media/anon/06bcf743-8b4d-409f-addc-520fc4e19299/PycharmProjects/LiveMe/venv1/lib/python3.6/site-packages/pymongo/bulk.py", line 329, in execute_command
raise BulkWriteError(full_result)
pymongo.errors.BulkWriteError: batch op errors occurred
messages.insert and messages.insert_many both insert 1 line and throw the error.

These files obviously do not contain properly formatted json - rather they contain a separate object on each line.
To turn them into valid json, you probably want a list of objects, i.e.:
[{ "channelType":"TEMPGROUP", ... },
{ "channelType":"TEMPGROUP", ... }]
You can achieve this by doing:
for i in raw_replay_data['data']['video_info']:
url3 = i['msgfile']
raw_message_data = urllib.request.urlopen(url3)
json_data = []
for line in raw_message_data:
json_data.append(json.loads(line))
messages.insert_one(json_data)
print(json_data)

1064 error python to mysql

I have a database of names and prices that gets updated daily when I run it through a batch of code and update a second database all of the names until it gets to 'aapl' at which point it throws a 1064 error which looks like this
-----------------------------------
Traceback (most recent call last):
File "testrun.PY", line 45, in <module>
t.Push.find_all(conn, cursor)
File "c:\tradetools.py", line 198, in find_all
Push.find_streak(conn, cursor, name)
File "c:\tradetools.py", line 189, in find_strea
k
.format(c, name))
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\cursors.py", line 166, in execute
result = self._query(query)
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\cursors.py", line 322, in _query
conn.query(q)
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\connections.py", line 837, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\connections.py", line 1021, in _read_query_result
result.read()
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\connections.py", line 1304, in read
first_packet = self.connection._read_packet()
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\connections.py", line 983, in _read_packet
packet.check_error()
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\connections.py", line 395, in check_error
err.raise_mysql_exception(self._data)
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\err.py", line 102, in raise_mysql_exception
raise errorclass(errno, errval)
pymysql.err.ProgrammingError: (1064, "42000You have an error in your SQL syntax;
check the manual that corresponds to your MySQL server version for the right sy
ntax to use near 'AAPL''' at line 1")
c:\>
The code that its running through looks like this, why does the update add more commas to the name appl?
def find_streak(conn, cursor, name):
print(name)
cursor.execute("SELECT * FROM `trade_data`.`import_data`"
" WHERE name =%s;", name)
logs = cursor.fetchall()
cursor.execute("INSERT IGNORE INTO `trade_data`.`analysis`(`name`) "
"VALUES (%s) ON DUPLICATE KEY UPDATE "
"ndays=0;", name)
conn.commit()
logs = [list(x) for x in logs]
logs.sort()
....
cursor.execute ("UPDATE `trade_data`.`analysis` "
"SET `ndays` = {0} WHERE name='{1}'"
.format(c, name))
conn.commit()
its pulling from a table that looks like this
date|name|price|
and entering into a table that is
date|name|result

neomodel giving Attribute error on save

I am running the following config for neo4j:
neo4j - 3.0.0
py2neo - 2.0.8
neomodel - 2.0.2
Finally, the code I try to run is:
class User(neomodel.StructuredNode):
user_id = neomodel.IntegerProperty(unique_index=True, required=True)
name = neomodel.StringProperty()
phone_number = neomodel.StringProperty()
user = User(user_id=6, name='Sourabh Dev', phone_number='9711237840').save()
I don't understand, why I keep getting this strange error. Am I doing something wrong here or should I use py2neo instead of neomodel?
My traceback is:
Traceback (most recent call last):
File "<input>", line 1, in <module>
File "/usr/local/lib/python3.4/site-packages/neomodel-2.0.2-py3.4.egg/neomodel/signals.py", line 25, in hooked
val = fn(self, *args, **kwargs)
File "/usr/local/lib/python3.4/site-packages/neomodel-2.0.2-py3.4.egg/neomodel/core.py", line 159, in save
self._id = self.create(self.__properties__)[0]._id
File "/usr/local/lib/python3.4/site-packages/neomodel-2.0.2-py3.4.egg/neomodel/core.py", line 289, in create
results = db.cypher_query(query, params)
File "/usr/local/lib/python3.4/site-packages/neomodel-2.0.2-py3.4.egg/neomodel/util.py", line 213, in cypher_query
results = self._execute_query(query, params)
File "/usr/local/lib/python3.4/site-packages/neomodel-2.0.2-py3.4.egg/neomodel/util.py", line 207, in _execute_query
results = self.session.cypher.execute(query, create_params=params)
File "/usr/local/lib/python3.4/site-packages/py2neo/cypher/core.py", line 136, in execute
results = tx.commit()
File "/usr/local/lib/python3.4/site-packages/py2neo/cypher/core.py", line 333, in commit
return self.post(self.__commit or self.__begin_commit)
File "/usr/local/lib/python3.4/site-packages/py2neo/cypher/core.py", line 288, in post
raise self.error_class.hydrate(error)
File "/usr/local/lib/python3.4/site-packages/py2neo/cypher/error/core.py", line 54, in hydrate
error_cls = getattr(error_module, title)
AttributeError: 'module' object has no attribute 'TypeError'

Two points here:
Existing Neomodel doesn't support the Neo4j 3.0
Cypher syntax has changed in 3.0, so the error raises.
In 2.x , MATCH n RETURN n
In 3.0 , MATCH (n) RETURN n
Node n is enclosed in braces.