c.execute('show tables')
for i in c:
print(i)
t=input('enter exact table name')
n=0
c.execute('show columns from {}'.format(t))
records=c.fetchall()
print(records)
for i in records:
n=n+1
a=1
data=''
while a!=0:
for i in range(n):
v=input("enter data")
if i==n-1:
data=data+v
else:
data=data+v+','
print(data)
print('insert into {} values{}'.format(t,data))
c.execute('insert into {} values({})'.format(t,data))
in this code i am getting the following error:
Traceback (most recent call last):
File "C:\Users\subra\OneDrive\Desktop\netflix.py", line 62, in <module>
create_input()
File "C:\Users\subra\OneDrive\Desktop\netflix.py", line 51, in create_input
c.execute('insert into {} values{}'.format(t,data))
File "C:\Users\subra\AppData\Local\Programs\Python\Python38\lib\site-packages\mysql\connector\cursor.py", line 569, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "C:\Users\subra\AppData\Local\Programs\Python\Python38\lib\site-packages\mysql\connector\connection.py", line 599, in cmd_query
result = self._handle_result(self._send_cmd(ServerCmd.QUERY, query))
File "C:\Users\subra\AppData\Local\Programs\Python\Python38\lib\site-packages\mysql\connector\connection.py", line 487, in _handle_result
raise errors.get_exception(packet)
mysql.connector.errors.ProgrammingError: 1054 (42S22): Unknown column 'subru' in 'field list'
can anyone help me out on how to avoid this error
i have also made sure that there is only 1 table and that is the one i am inserting
this is the output that i get including the input vales:
>>>('hellopython',)
>>>enter exact table namehellopython
>>>[('student', b'varchar(30)', 'YES', '', None, ''), ('rollno', b'varchar(30)', 'YES', '', None,'')]
>>>enter datasubru
>>>enter data29
>>>(subru,29)
>>>insert into hellopython values(subru,29)
***Traceback (most recent call last):
File "C:\Users\subra\OneDrive\Desktop\netflix.py", line 62, in <module>
create_input()
File "C:\Users\subra\OneDrive\Desktop\netflix.py", line 51, in create_input
c.execute('insert into {} values{}'.format(t,data))
File "C:\Users\subra\AppData\Local\Programs\Python\Python38\lib\site-packages\mysql\connector\cursor.py", line 569, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "C:\Users\subra\AppData\Local\Programs\Python\Python38\lib\site-packages\mysql\connector\connection.py", line 599, in cmd_query
result = self._handle_result(self._send_cmd(ServerCmd.QUERY, query))
File "C:\Users\subra\AppData\Local\Programs\Python\Python38\lib\site-packages\mysql\connector\connection.py", line 487, in _handle_result
raise errors.get_exception(packet)
mysql.connector.errors.ProgrammingError: 1054 (42S22): Unknown column 'subru' in 'field list'***
You are missing quotes around the data.
So MySQL thinks that subru is not a name. instead he thninks it is a column name
if i==n-1:
data=data+"'"+v+ "'"
else:
data=data+"'"+v+ "'"+','
print(data)
print('insert into {} values{}'.format(t,data))
c.execute('insert into {} values({})'.format(t,data))
Related
I have a inquiry from my job to create 2 tables related one-to-one and insert some rows on the table. I have a .CSV with the data dictionary (column name and data type) from the table and I am wondering to know how to declare the tables columns automatically (declarative syntax) without write one by one column (there are 260 columns). Same thing for the insert, how to add rows to the multiple columns table without write column by column?
I have the data in a Data frame but I was not able to insert it using df.to_sql from pandas. Do you guys have any similar example?
The database used is MySQL. The table structure in the database is showed below:
enter image description here
Below is what I did.
I have created a function to define the table constraints.
def create_mysql_tables(dataframe, engine):
# print(dataframe.iloc)
if 'Active' in dataframe.columns:
start = time.time()
engine.execute('DROP TABLE IF EXISTS com_treb;')
engine.execute('DROP TABLE IF EXISTS cnd_treb;')
engine.execute('DROP TABLE IF EXISTS res_treb;')
engine.execute('DROP TABLE IF EXISTS mls_treb;')
dataframe.to_sql("mls_treb", if_exists='replace', con=engine,
dtype={'mls_number': VARCHAR(dataframe.index.get_level_values('mls_number').str.len().max())})
with engine.connect() as con:
con.execute('ALTER TABLE `mls_treb` ADD PRIMARY KEY (`mls_number`);')
end = time.time()
print(end - start)
First questions is, there is a better way to define a primary key using the SQLALCHEMY avoiding to write the query?
After I have created the table I have to insert some row if it doesn't exist. I was trying to do it by using the function below.
def crud_database(engine, mls_full_dataframe, res_full_dataframe):
for index, row in mls_full_dataframe.iterrows():
row_db_mls = pd.read_sql_query("SELECT * FROM mls_treb WHERE `mls_number` LIKE %(mlsnumber)s ", engine, params={'mlsnumber' : row.at['mls_number']})
if row_db_mls.empty:
row.to_sql("mls_treb", if_exists='append', con=engine, index_label='mls_number' )
if row.at['Class'][0] == 'RES':
row_res = res_full_dataframe.iloc[row.at['mls_number']]
else:
print(0)
but I am getting the error below when I am trying to insert the row from picture below:
image from row to be inserted with index 0
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1782, in _execute_context
self.dialect.do_executemany(
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 729, in do_executemany
cursor.executemany(statement, parameters)
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/mysql/connector/cursor.py", line 670, in executemany
return self.execute(stmt)
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/mysql/connector/cursor.py", line 568, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/mysql/connector/connection.py", line 854, in cmd_query
result = self._handle_result(self._send_cmd(ServerCmd.QUERY, query))
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/mysql/connector/connection.py", line 664, in _handle_result
raise errors.get_exception(packet)
mysql.connector.errors.ProgrammingError: 1054 (42S22): Unknown column '3' in 'field list'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/pandas/io/sql.py", line 1419, in to_sql
raise err
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/pandas/io/sql.py", line 1411, in to_sql
table.insert(chunksize, method=method)
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/pandas/io/sql.py", line 845, in insert
exec_insert(conn, keys, chunk_iter)
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/pandas/io/sql.py", line 762, in _execute_insert
conn.execute(self.table.insert(), data)
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1289, in execute
return meth(self, multiparams, params, _EMPTY_EXECUTION_OPTS)
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/sqlalchemy/sql/elements.py", line 325, in _execute_on_connection
return connection._execute_clauseelement(
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1481, in _execute_clauseelement
ret = self._execute_context(
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1845, in _execute_context
self._handle_dbapi_exception(
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2026, in _handle_dbapi_exception
util.raise_(
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1782, in _execute_context
self.dialect.do_executemany(
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 729, in do_executemany
cursor.executemany(statement, parameters)
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/mysql/connector/cursor.py", line 670, in executemany
return self.execute(stmt)
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/mysql/connector/cursor.py", line 568, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/mysql/connector/connection.py", line 854, in cmd_query
result = self._handle_result(self._send_cmd(ServerCmd.QUERY, query))
File "/Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages/mysql/connector/connection.py", line 664, in _handle_result
raise errors.get_exception(packet)
sqlalchemy.exc.ProgrammingError: (mysql.connector.errors.ProgrammingError) 1054 (42S22): Unknown column '3' in 'field list'
[SQL: INSERT INTO mls_treb (mls_number, `3`) VALUES (%(mls_number)s, %(3)s)]
[parameters: ({'mls_number': 'mls_number', '3': 'N5404949'}, {'mls_number': 'active', '3': True}, {'mls_number': 'class_name', '3': 'RES'}, {'mls_number': 'active_date', '3': datetime.date(2022, 1, 16)})]
(Background on this error at: https://sqlalche.me/e/14/f405)
With Gord Thompsom help I could find the answer for problem. Prior to use the row to insert
row_to_insert = mls_full_dataframe.loc[mls_full_dataframe['mls_number'] == row.at['mls_number']]
row_to_insert.to_sql("mls_treb", if_exists='append', con=engine, index=False)
I was getting a series instead of row as dataframe. So, when I was trying to insert the row as serie, the number '3' was the index used as key and the compiler did not find the column name properly with that name "3". To fix it, I was retrieving the row_to_insert as dataframe and trying to insert it into database.
My Pandas Dataframe named as data_five_minutes:
script_id
date_time
open
0
1
2019-01-11 09:35:00
25
1
1
2019-01-11 09:40:00
30
2
1
2019-01-11 09:45:00
48
Full ss:
Now I was trying to get only data which are having script_id as 1:
data = ps.sqldf(f"""SELECT d4.script_id,d4.date_time,d4.open,d4.high,d4.low,d4.close,d4.volume
FROM data_five_minutes d4 WHERE d4.script_id = {id}""")
this is just small condition but in original it has nested where clause so SQL query will be passed on original code as earlier i was passing the same query in database.
I am getting below error:
Traceback (most recent call last): File
"C:\Python\lib\site-packages\sqlalchemy\engine\base.py", line 1752, in
_e xecute_context
cursor, statement, parameters, context File "C:\Python\lib\site-packages\sqlalchemy\engine\default.py", line 714,
in do_executemany
cursor.executemany(statement, parameters) sqlite3.InterfaceError: Error binding parameter 2 - probably unsupported type.
The above exception was the direct cause of the following exception:
Traceback (most recent call last): File
"D:\python_projects\auto\testing.py", line 70, in
FROM data_five_minutes d4 WHERE d4.script_id = 1""") File "C:\Python\lib\site-packages\pandasql\sqldf.py", line 156, in sqldf
return PandaSQL(db_uri)(query, env) File "C:\Python\lib\site-packages\pandasql\sqldf.py", line 58, in call
write_table(env[table_name], table_name, conn) File "C:\Python\lib\site-packages\pandasql\sqldf.py", line 121, in
write_table
index=not any(name is None for name in df.index.names)) # load index into d b if all levels are named File
"C:\Python\lib\site-packages\pandas\io\sql.py", line 518, in to_sql
method=method, File "C:\Python\lib\site-packages\pandas\io\sql.py", line 1320, in to_sql
table.insert(chunksize, method=method) File "C:\Python\lib\site-packages\pandas\io\sql.py", line 756, in insert
exec_insert(conn, keys, chunk_iter) File "C:\Python\lib\site-packages\pandas\io\sql.py", line 670, in
_execute_ins ert
conn.execute(self.table.insert(), data) File "C:\Python\lib\site-packages\sqlalchemy\engine\base.py", line 1263, in
ex ecute
return meth(self, multiparams, params, _EMPTY_EXECUTION_OPTS) File "C:\Python\lib\site-packages\sqlalchemy\sql\elements.py", line
324, in _e xecute_on_connection
self, multiparams, params, execution_options File "C:\Python\lib\site-packages\sqlalchemy\engine\base.py", line 1462, in
_e xecute_clauseelement
cache_hit=cache_hit, File "C:\Python\lib\site-packages\sqlalchemy\engine\base.py", line 1815, in
e xecute_context
e, statement, parameters, cursor, context File "C:\Python\lib\site-packages\sqlalchemy\engine\base.py", line 1996, in
h andle_dbapi_exception
sqlalchemy_exception, with_traceback=exc_info[2], from=e File "C:\Python\lib\site-packages\sqlalchemy\util\compat.py", line 207, in
rai se
raise exception File "C:\Python\lib\site-packages\sqlalchemy\engine\base.py", line 1752, in
_e xecute_context
cursor, statement, parameters, context File "C:\Python\lib\site-packages\sqlalchemy\engine\default.py", line 714,
in do_executemany
cursor.executemany(statement, parameters) sqlalchemy.exc.InterfaceError: (sqlite3.InterfaceError) Error binding
parameter 2 - probably unsupported type. [SQL: INSERT INTO
data_five_minutes (script_id, date_time, open, high, low, clos e,
volume) VALUES (?, ?, ?, ?, ?, ?, ?)] [parameters: ((1, '2021-05-25
10:30:00.000000', Decimal('1978.6'), Decimal('1985 '),
Decimal('1978.1'), Decimal('1985'), Decimal('323')), (1, '2021-05-25
10:35:0
0.000000', Decimal('1985'), Decimal('1986.05'), Decimal('1982.75'), Decimal('198
3.85'), Decimal('954')), and so on...] (Background on this error at: https://sqlalche.me/e/14/rvf5)
Process returned 1 (0x1) execution time : 2.672 s Press any key
to continue . . .
Using the mysql.connector package (with Django), and executing:
c.execute("""
select *
from shop_sales
where product_id in %s
""", [(83, 84, 85, 87, 88, 89)])
We get the following traceback:
Traceback (most recent call last):
File "/srv/venv/dev35/lib/python3.5/site-packages/mysql/connector/conversion.py", line 179, in to_mysql
return getattr(self, "_{0}_to_mysql".format(type_name))(value)
AttributeError: 'DjangoMySQLConverter' object has no attribute '_tuple_to_mysql'
This is reported as a bug at https://bugs.mysql.com/bug.php?id=89112, but I'm trying to find a workaround by writing a converter (from my settings.py):
from mysql.connector.django.base import DjangoMySQLConverter
def _tuple_to_mysql(self, value):
res = []
for item in value:
msval = self.to_mysql(item)
if not isinstance(msval, bytes):
msval = str(msval).encode()
res.append(msval)
return b'(' + b', '.join(res) + b')'
DjangoMySQLConverter._tuple_to_mysql = _tuple_to_mysql
but it looks like mysql.connector is adding extra quotes around it:
Traceback (most recent call last):
File "/srv/venv/dev35/lib/python3.5/site-packages/mysql/connector/django/base.py", line 176, in _execute_wrapper
return method(query, args)
File "/srv/venv/dev35/lib/python3.5/site-packages/mysql/connector/cursor.py", line 561, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "/srv/venv/dev35/lib/python3.5/site-packages/mysql/connector/connection.py", line 525, in cmd_query
result = self._handle_result(self._send_cmd(ServerCmd.QUERY, query))
File "/srv/venv/dev35/lib/python3.5/site-packages/mysql/connector/connection.py", line 427, in _handle_result
raise errors.get_exception(packet)
mysql.connector.errors.ProgrammingError: 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near ''(83, 84, 85, 87, 88, 89)'
What am I doing wrong?
I just can't see a reason for this error. I have tried the same SQL in phpMyAdmin and it works perfectly fine, but fails when trying from Python.
Python code with SQL query:
cursor.execute("UPDATE marketPricesAvg SET avg%sh=(SELECT AVG(price) FROM marketPrices WHERE itemName = %s AND ((NOW() - marketPrices.datetime) < %s) WHERE itemName = %s)", (time, itemName, time_sec, itemName))
Error message:
Traceback (most recent call last):
File "/root/marketprices/insert.py", line 42, in <module>
calculate_avg(itemName, time)
File "/root/marketprices/insert.py", line 29, in calculate_avg
cursor.execute("UPDATE marketPricesAvg SET avg%sh=(SELECT AVG(price) FROM marketPrices WHERE itemName = %s AND ((NOW() - marketPrices.datetime) < %s) WHERE itemName = %s)", (time, itemName, time_sec, itemName))
File "/usr/local/lib/python2.7/dist-packages/pymysql/cursors.py", line 170, in execute
result = self._query(query)
File "/usr/local/lib/python2.7/dist-packages/pymysql/cursors.py", line 328, in _query
conn.query(q)
File "/usr/local/lib/python2.7/dist-packages/pymysql/connections.py", line 516, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/usr/local/lib/python2.7/dist-packages/pymysql/connections.py", line 727, in _read_query_result
result.read()
File "/usr/local/lib/python2.7/dist-packages/pymysql/connections.py", line 1066, in read
first_packet = self.connection._read_packet()
File "/usr/local/lib/python2.7/dist-packages/pymysql/connections.py", line 683, in _read_packet
packet.check_error()
File "/usr/local/lib/python2.7/dist-packages/pymysql/protocol.py", line 220, in check_error
err.raise_mysql_exception(self._data)
File "/usr/local/lib/python2.7/dist-packages/pymysql/err.py", line 109, in raise_mysql_exception
raise errorclass(errno, errval)
pymysql.err.ProgrammingError: (1064, u"You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'WHERE itemName = 'Aluminium')' at line 1")
Thank you for any ideas.
Your query fully qualified would look something like
UPDATE marketPricesAvg
SET avg1234 = (
SELECT AVG(price) FROM marketPrices
WHERE itemName = 'Aluminium' AND
((NOW() - marketPrices.datetime) < '100') WHERE itemName = 'Aluminium'
);
Edit:
Should be
(NOW() - marketPrices.datetime) < %s) WHERE itemName = %s
The final parentheses is misplaced
I have a database of names and prices that gets updated daily when I run it through a batch of code and update a second database all of the names until it gets to 'aapl' at which point it throws a 1064 error which looks like this
-----------------------------------
Traceback (most recent call last):
File "testrun.PY", line 45, in <module>
t.Push.find_all(conn, cursor)
File "c:\tradetools.py", line 198, in find_all
Push.find_streak(conn, cursor, name)
File "c:\tradetools.py", line 189, in find_strea
k
.format(c, name))
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\cursors.py", line 166, in execute
result = self._query(query)
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\cursors.py", line 322, in _query
conn.query(q)
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\connections.py", line 837, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\connections.py", line 1021, in _read_query_result
result.read()
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\connections.py", line 1304, in read
first_packet = self.connection._read_packet()
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\connections.py", line 983, in _read_packet
packet.check_error()
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\connections.py", line 395, in check_error
err.raise_mysql_exception(self._data)
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\err.py", line 102, in raise_mysql_exception
raise errorclass(errno, errval)
pymysql.err.ProgrammingError: (1064, "42000You have an error in your SQL syntax;
check the manual that corresponds to your MySQL server version for the right sy
ntax to use near 'AAPL''' at line 1")
c:\>
The code that its running through looks like this, why does the update add more commas to the name appl?
def find_streak(conn, cursor, name):
print(name)
cursor.execute("SELECT * FROM `trade_data`.`import_data`"
" WHERE name =%s;", name)
logs = cursor.fetchall()
cursor.execute("INSERT IGNORE INTO `trade_data`.`analysis`(`name`) "
"VALUES (%s) ON DUPLICATE KEY UPDATE "
"ndays=0;", name)
conn.commit()
logs = [list(x) for x in logs]
logs.sort()
....
cursor.execute ("UPDATE `trade_data`.`analysis` "
"SET `ndays` = {0} WHERE name='{1}'"
.format(c, name))
conn.commit()
its pulling from a table that looks like this
date|name|price|
and entering into a table that is
date|name|result