I have a Flask based backend application and when running the code locally, this error does not happen, but it does happen when deployed in my server in a kubernetes pod.
Interestingly enough, when you run the code for the first time, it fails instantly (when running the Future) but when you run it again the error still happens, but the code continues (and completes successfully). Pretty bizarre.
The error I am getting is:
2021-07-30 14:59:23,537 - mit_backend.logic.optimize - INFO - Inside first pass run code
2021-07-30 14:59:23,587 - mit_backend.logic.optimize - INFO - Optimising region ML10F
2021-07-30 14:59:23,588 - mit_backend.logic.optimize - INFO - Optimising region ML47F
2021-07-30 14:59:23,589 - mit_backend.logic.optimize - INFO - Optimising region ML40F
--- Logging error ---
concurrent.futures.process._RemoteTraceback:
"""
Traceback (most recent call last):
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1277, in _execute_context
cursor, statement, parameters, context
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute
cursor.execute(statement, parameters)
psycopg2.OperationalError: SSL error: wrong version number
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/lib64/python3.6/concurrent/futures/process.py", line 175, in _process_worker
r = call_item.fn(*call_item.args, **call_item.kwargs)
File "/code/mit_backend/logic/optimize.py", line 241, in optimize_first_pass_run
config = get_csv_config(user_identity)
File "/code/mit_backend/modules/v1/__init__.py", line 623, in get_csv_config
res = CSVConfig.query.filter(CSVConfig.user_id == user_identity).first()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/query.py", line 3429, in first
ret = list(self[0:1])
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/query.py", line 3203, in __getitem__
return list(res)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/query.py", line 3535, in __iter__
return self._execute_and_instances(context)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/query.py", line 3560, in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1011, in execute
return meth(self, multiparams, params)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1130, in _execute_clauseelement
distilled_params,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1317, in _execute_context
e, statement, parameters, cursor, context
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1511, in _handle_dbapi_exception
sqlalchemy_exception, with_traceback=exc_info[2], from_=e
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 182, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1277, in _execute_context
cursor, statement, parameters, context
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) SSL error: wrong version number
[SQL: SELECT csv_config.user_id AS csv_config_user_id, csv_config."csvHeaders" AS "csv_config_csvHeaders", csv_config."deliveryName" AS "csv_config_deliveryName", csv_config.id AS csv_config_id, csv_config.line1 AS csv_config_line1, csv_config.line2 AS csv_config_line2, csv_config.quantity AS csv_config_quantity, csv_config."routeNumber" AS "csv_config_routeNumber", csv_config.suburb AS csv_config_suburb, csv_config.weight AS csv_config_weight
FROM csv_config
WHERE csv_config.user_id = %(user_id_1)s
LIMIT %(param_1)s]
[parameters: {'user_id_1': 'sftp', 'param_1': 1}]
(Background on this error at: http://sqlalche.me/e/13/e3q8)
"""
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/code/mit_backend/logic/optimize.py", line 300, in first_pass_run
future_df_result = future.result()
File "/usr/lib64/python3.6/concurrent/futures/_base.py", line 425, in result
return self.__get_result()
File "/usr/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result
raise self._exception
sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) SSL error: wrong version number
[SQL: SELECT csv_config.user_id AS csv_config_user_id, csv_config."csvHeaders" AS "csv_config_csvHeaders", csv_config."deliveryName" AS "csv_config_deliveryName", csv_config.id AS csv_config_id, csv_config.line1 AS csv_config_line1, csv_config.line2 AS csv_config_line2, csv_config.quantity AS csv_config_quantity, csv_config."routeNumber" AS "csv_config_routeNumber", csv_config.suburb AS csv_config_suburb, csv_config.weight AS csv_config_weight
FROM csv_config
WHERE csv_config.user_id = %(user_id_1)s
LIMIT %(param_1)s]
[parameters: {'user_id_1': 'sftp', 'param_1': 1}]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib64/python3.6/logging/__init__.py", line 994, in emit
msg = self.format(record)
File "/usr/lib64/python3.6/logging/__init__.py", line 840, in format
return fmt.format(record)
File "/usr/lib64/python3.6/logging/__init__.py", line 577, in format
record.message = record.getMessage()
File "/usr/lib64/python3.6/logging/__init__.py", line 338, in getMessage
msg = msg % self.args
TypeError: not all arguments converted during string formatting
Call stack:
File "/usr/lib64/python3.6/threading.py", line 884, in _bootstrap
self._bootstrap_inner()
File "/usr/lib64/python3.6/threading.py", line 916, in _bootstrap_inner
self.run()
File "/usr/lib64/python3.6/threading.py", line 864, in run
self._target(*self._args, **self._kwargs)
File "/usr/lib64/python3.6/socketserver.py", line 654, in process_request_thread
self.finish_request(request, client_address)
File "/usr/lib64/python3.6/socketserver.py", line 364, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/usr/lib64/python3.6/socketserver.py", line 724, in __init__
self.handle()
File "/usr/local/lib/python3.6/site-packages/werkzeug/serving.py", line 345, in handle
BaseHTTPRequestHandler.handle(self)
File "/usr/lib64/python3.6/http/server.py", line 418, in handle
self.handle_one_request()
File "/usr/local/lib/python3.6/site-packages/werkzeug/serving.py", line 379, in handle_one_request
return self.run_wsgi()
File "/usr/local/lib/python3.6/site-packages/werkzeug/serving.py", line 323, in run_wsgi
execute(self.server.app)
File "/usr/local/lib/python3.6/site-packages/werkzeug/serving.py", line 312, in execute
application_iter = app(environ, start_response)
File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 2464, in __call__
return self.wsgi_app(environ, start_response)
File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 2447, in wsgi_app
response = self.full_dispatch_request()
File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1950, in full_dispatch_request
rv = self.dispatch_request()
File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1936, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/usr/local/lib/python3.6/site-packages/flask_restx/api.py", line 375, in wrapper
resp = resource(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/flask/views.py", line 89, in view
return self.dispatch_request(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/flask_restx/resource.py", line 44, in dispatch_request
resp = meth(*args, **kwargs)
File "/code/mit_backend/modules/v1/controllers/manifest.py", line 161, in post
files, df, sftp_date_latest = do_optimize(addresses, user_identity)
File "/code/mit_backend/logic/optimize.py", line 38, in do_optimize
df = first_pass_run(df, user_identity)
File "/code/mit_backend/logic/optimize.py", line 307, in first_pass_run
LOG.error("Exception is ", e.__cause__)
Message: 'Exception is '
Arguments: (_RemoteTraceback('\n"""\nTraceback (most recent call last):\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1277, in _execute_context\n cursor, statement, parameters, context\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute\n cursor.execute(statement, parameters)\npsycopg2.OperationalError: SSL error: decryption failed or bad record mac\n\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File "/usr/lib64/python3.6/concurrent/futures/process.py", line 175, in _process_worker\n r = call_item.fn(*call_item.args, **call_item.kwargs)\n File "/code/mit_backend/logic/optimize.py", line 241, in optimize_first_pass_run\n config = get_csv_config(user_identity)\n File "/code/mit_backend/modules/v1/__init__.py", line 623, in get_csv_config\n res = CSVConfig.query.filter(CSVConfig.user_id == user_identity).first()\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/query.py", line 3429, in first\n ret = list(self[0:1])\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/query.py", line 3203, in __getitem__\n return list(res)\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/query.py", line 3535, in __iter__\n return self._execute_and_instances(context)\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/query.py", line 3560, in _execute_and_instances\n result = conn.execute(querycontext.statement, self._params)\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1011, in execute\n return meth(self, multiparams, params)\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection\n return connection._execute_clauseelement(self, multiparams, params)\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1130, in _execute_clauseelement\n distilled_params,\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1317, in _execute_context\n e, statement, parameters, cursor, context\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1511, in _handle_dbapi_exception\n sqlalchemy_exception, with_traceback=exc_info[2], from_=e\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 182, in raise_\n raise exception\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1277, in _execute_context\n cursor, statement, parameters, context\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute\n cursor.execute(statement, parameters)\nsqlalchemy.exc.OperationalError: (psycopg2.OperationalError) SSL error: decryption failed or bad record mac\n\n[SQL: SELECT csv_config.user_id AS csv_config_user_id, csv_config."csvHeaders" AS "csv_config_csvHeaders", csv_config."deliveryName" AS "csv_config_deliveryName", csv_config.id AS csv_config_id, csv_config.line1 AS csv_config_line1, csv_config.line2 AS csv_config_line2, csv_config.quantity AS csv_config_quantity, csv_config."routeNumber" AS "csv_config_routeNumber", csv_config.suburb AS csv_config_suburb, csv_config.weight AS csv_config_weight \nFROM csv_config \nWHERE csv_config.user_id = %(user_id_1)s \n LIMIT %(param_1)s]\n[parameters: {\'user_id_1\': \'sftp\', \'param_1\': 1}]\n(Background on this error at: http://sqlalche.me/e/13/e3q8)\n"""',),)
The error happens inside a futures ProcessPoolExecutor:
def first_pass_run(df, user_identity):
LOG.info("Inside first pass run code")
first_json = True
df_final = None
df_grouped = df.groupby('Route Number')
auth_claim = get_authorization_claims_from_header()
with concurrent.futures.ProcessPoolExecutor(max_workers=8) as executor:
_futures = []
for region_name, _df in df_grouped:
_futures.append(
executor.submit(
optimize_first_pass_run, region_name, _df, user_identity, auth_claim
))
for future in concurrent.futures.as_completed(_futures):
try:
future_df_result = future.result()
if first_json:
df_final = future_df_result
first_json = False
else:
df_final = df_final.append(future_df_result, ignore_index=True)
except Exception as e:
LOG.error("Exception is ", e.__cause__)
I start my SQLAlchemy engine in a pessimistic fashion:
pg_db = SQLAlchemy(app)
engine = create_engine(app.config['SQLALCHEMY_DATABASE_URI'], pool_pre_ping=True)
The code is not running on WSGI or gunicorn, it's just as simple as:
if __name__ == '__main__':
LOG.info('running environment: %s', os.environ.get('ENV', 'production'))
app.config['DEBUG'] = os.environ.get('ENV') == 'development'
app.run(host='0.0.0.0', port=5001, debug=False, threaded=True)
Any clues why this might be happening? Thank you.
I haven't been able to find a proper fix for this, but I implemented the following things that although the exception is still being thrown, the code retries and ends up succeeding:
Removing the connection pool
engine = create_engine(app.config['SQLALCHEMY_DATABASE_URI'], poolclass=NullPool)
Event listeners
#event.listens_for(engine, "connect")
def connect(dbapi_connection, connection_record):
connection_record.info['pid'] = os.getpid()
#event.listens_for(engine, "checkout")
def checkout(dbapi_connection, connection_record, connection_proxy):
pid = os.getpid()
if connection_record.info['pid'] != pid:
connection_record.connection = connection_proxy.connection = None
raise exc.DisconnectionError(
"Connection record belongs to pid %s, "
"attempting to check out in pid %s" %
(connection_record.info['pid'], pid)
)
but, as per the documentation, the two definitions above relate to an optimistic way of handling the connection and using Pools
Reconnecting before initializing the Process Pool
engine.dispose()
with concurrent.futures.ProcessPoolExecutor(max_workers=8) as executor:
_futures = []
for region_name, _df in df_grouped:
_futures.append(
executor.submit(
optimize_first_pass_run, region_name, _df, user_identity, auth_claim, True
))
for future in concurrent.futures.as_completed(_futures):
future_df_result = future.result()
Adding retries when trying to insert objects in the database
def add_new_record(row):
attempts = 0
while attempts <= 5:
attempts = attempts + 1
try:
pg_db.session.add(row)
pg_db.session.commit()
return True
except (SQLAlchemyError, psycopg2.OperationalError, sqlalchemy.exc.OperationalError) as e:
if attempts < 5:
LOG.warn("Attempt failed. Trying rollback")
pg_db.session.rollback()
LOG.warn(e.__cause__)
LOG.warn(e.__str__())
time.sleep(5)
else:
LOG.error("Maximum number of retries reached. Raising an error")
raise e
return False
Using the mysql.connector package (with Django), and executing:
c.execute("""
select *
from shop_sales
where product_id in %s
""", [(83, 84, 85, 87, 88, 89)])
We get the following traceback:
Traceback (most recent call last):
File "/srv/venv/dev35/lib/python3.5/site-packages/mysql/connector/conversion.py", line 179, in to_mysql
return getattr(self, "_{0}_to_mysql".format(type_name))(value)
AttributeError: 'DjangoMySQLConverter' object has no attribute '_tuple_to_mysql'
This is reported as a bug at https://bugs.mysql.com/bug.php?id=89112, but I'm trying to find a workaround by writing a converter (from my settings.py):
from mysql.connector.django.base import DjangoMySQLConverter
def _tuple_to_mysql(self, value):
res = []
for item in value:
msval = self.to_mysql(item)
if not isinstance(msval, bytes):
msval = str(msval).encode()
res.append(msval)
return b'(' + b', '.join(res) + b')'
DjangoMySQLConverter._tuple_to_mysql = _tuple_to_mysql
but it looks like mysql.connector is adding extra quotes around it:
Traceback (most recent call last):
File "/srv/venv/dev35/lib/python3.5/site-packages/mysql/connector/django/base.py", line 176, in _execute_wrapper
return method(query, args)
File "/srv/venv/dev35/lib/python3.5/site-packages/mysql/connector/cursor.py", line 561, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "/srv/venv/dev35/lib/python3.5/site-packages/mysql/connector/connection.py", line 525, in cmd_query
result = self._handle_result(self._send_cmd(ServerCmd.QUERY, query))
File "/srv/venv/dev35/lib/python3.5/site-packages/mysql/connector/connection.py", line 427, in _handle_result
raise errors.get_exception(packet)
mysql.connector.errors.ProgrammingError: 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near ''(83, 84, 85, 87, 88, 89)'
What am I doing wrong?
I am trying to push the excel file by using to_sql function and using data frame after transpose the dataframe...
and this is my code looks like
import pandas as pd
import os
import sqlalchemy
# MySQL Connection
MYSQL_USER = 'xxxxx'
MYSQL_PASSWORD = 'xxxxxxxx'
MYSQL_HOST_IP = '127.0.0.1'
MYSQL_PORT = 3306
MYSQL_DATABASE = 'xlsx_test_db'
# connect db
engine = sqlalchemy.create_engine('mysql+mysqlconnector://' + MYSQL_USER + ':' + MYSQL_PASSWORD + '#' + MYSQL_HOST_IP + ':' + str(
MYSQL_PORT) + '/' + MYSQL_DATABASE, echo=False)
engine.connect()
mydir = (os.getcwd()).replace('\\', '/') + '/'
raw_lte = pd.read_excel(r'' + mydir + 'MNM_Rotterdam_5_Daily_Details-20191216081027.xlsx', sheet_name='raw_4G')
dft = raw_lte.T
dft.columns = dft.iloc[0]
dft = dft.iloc[1:]
# reading and insert one file at a time
for file in os.listdir('.'):
# only process excels files
file_basename, extension = file.split('.')
if extension == 'xlsx':
dft.to_sql(file_basename.lower(), con=engine, if_exists='replace')
and this the error
Traceback
Traceback (most recent call last):
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1193, in _execute_context
context)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\engine\default.py", line 507, in do_execute
cursor.execute(statement, parameters)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\mysql\connector\cursor.py", line 551, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\mysql\connector\connection.py", line 490, in cmd_query
result = self._handle_result(self._send_cmd(ServerCmd.QUERY, query))
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\mysql\connector\connection.py", line 395, in _handle_result
raise errors.get_exception(packet)
mysql.connector.errors.ProgrammingError: 1170 (42000): BLOB/TEXT column 'index' used in key specification without a key length
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:/Users/DELL/PycharmProjects/automateDB/swap.py", line 36, in <module>
dft.to_sql(file_basename.lower(), con=engine, if_exists='replace')
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\pandas\core\generic.py", line 2532, in to_sql
dtype=dtype, method=method)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\pandas\io\sql.py", line 460, in to_sql
chunksize=chunksize, dtype=dtype, method=method)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\pandas\io\sql.py", line 1173, in to_sql
table.create()
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\pandas\io\sql.py", line 585, in create
self._execute_create()
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\pandas\io\sql.py", line 569, in _execute_create
self.table.create()
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\sql\schema.py", line 778, in create
checkfirst=checkfirst)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1940, in _run_visitor
conn._run_visitor(visitorcallable, element, **kwargs)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1549, in _run_visitor
**kwargs).traverse_single(element)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\sql\visitors.py", line 121, in traverse_single
return meth(obj, **kw)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\sql\ddl.py", line 796, in visit_table
self.traverse_single(index)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\sql\visitors.py", line 121, in traverse_single
return meth(obj, **kw)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\sql\ddl.py", line 823, in visit_index
self.connection.execute(CreateIndex(index))
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\engine\base.py", line 948, in execute
return meth(self, multiparams, params)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\sql\ddl.py", line 68, in _execute_on_connection
return connection._execute_ddl(self, multiparams, params)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1009, in _execute_ddl
compiled
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1200, in _execute_context
context)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1413, in _handle_dbapi_exception
exc_info
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\util\compat.py", line 203, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\util\compat.py", line 186, in reraise
raise value.with_traceback(tb)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1193, in _execute_context
context)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\engine\default.py", line 507, in do_execute
cursor.execute(statement, parameters)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\mysql\connector\cursor.py", line 551, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\mysql\connector\connection.py", line 490, in cmd_query
result = self._handle_result(self._send_cmd(ServerCmd.QUERY, query))
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\mysql\connector\connection.py", line 395, in _handle_result
raise errors.get_exception(packet)
sqlalchemy.exc.ProgrammingError: (mysql.connector.errors.ProgrammingError) 1170 (42000): BLOB/TEXT column 'index' used in key specification without a key length [SQL: 'CREATE INDEX `ix_mnm_rotterdam_5_daily_details-20191216081027_index` ON `mnm_rotterdam_5_daily_details-20191216081027` (`index`)'] (Background on this error at: http://sqlalche.me/e/f405)
I think there's a problem with the excel format, but I don't know to solve this issue
Note:
I tried to use this
raw_4G.to_sql(file_basename.lower(), con=engine, if_exists='replace')
insted of
dft.to_sql(file_basename.lower(), con=engine, if_exists='replace')
It works but it gives me error in the run time with this
Traceback (most recent call last):
File "C:/Users/DELL/PycharmProjects/automateDB/swap.py", line 34, in <module>
file_basename, extension = file.split('.')
ValueError: not enough values to unpack (expected 2, got 1)
I just can't see a reason for this error. I have tried the same SQL in phpMyAdmin and it works perfectly fine, but fails when trying from Python.
Python code with SQL query:
cursor.execute("UPDATE marketPricesAvg SET avg%sh=(SELECT AVG(price) FROM marketPrices WHERE itemName = %s AND ((NOW() - marketPrices.datetime) < %s) WHERE itemName = %s)", (time, itemName, time_sec, itemName))
Error message:
Traceback (most recent call last):
File "/root/marketprices/insert.py", line 42, in <module>
calculate_avg(itemName, time)
File "/root/marketprices/insert.py", line 29, in calculate_avg
cursor.execute("UPDATE marketPricesAvg SET avg%sh=(SELECT AVG(price) FROM marketPrices WHERE itemName = %s AND ((NOW() - marketPrices.datetime) < %s) WHERE itemName = %s)", (time, itemName, time_sec, itemName))
File "/usr/local/lib/python2.7/dist-packages/pymysql/cursors.py", line 170, in execute
result = self._query(query)
File "/usr/local/lib/python2.7/dist-packages/pymysql/cursors.py", line 328, in _query
conn.query(q)
File "/usr/local/lib/python2.7/dist-packages/pymysql/connections.py", line 516, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/usr/local/lib/python2.7/dist-packages/pymysql/connections.py", line 727, in _read_query_result
result.read()
File "/usr/local/lib/python2.7/dist-packages/pymysql/connections.py", line 1066, in read
first_packet = self.connection._read_packet()
File "/usr/local/lib/python2.7/dist-packages/pymysql/connections.py", line 683, in _read_packet
packet.check_error()
File "/usr/local/lib/python2.7/dist-packages/pymysql/protocol.py", line 220, in check_error
err.raise_mysql_exception(self._data)
File "/usr/local/lib/python2.7/dist-packages/pymysql/err.py", line 109, in raise_mysql_exception
raise errorclass(errno, errval)
pymysql.err.ProgrammingError: (1064, u"You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'WHERE itemName = 'Aluminium')' at line 1")
Thank you for any ideas.
Your query fully qualified would look something like
UPDATE marketPricesAvg
SET avg1234 = (
SELECT AVG(price) FROM marketPrices
WHERE itemName = 'Aluminium' AND
((NOW() - marketPrices.datetime) < '100') WHERE itemName = 'Aluminium'
);
Edit:
Should be
(NOW() - marketPrices.datetime) < %s) WHERE itemName = %s
The final parentheses is misplaced
I have a database of names and prices that gets updated daily when I run it through a batch of code and update a second database all of the names until it gets to 'aapl' at which point it throws a 1064 error which looks like this
-----------------------------------
Traceback (most recent call last):
File "testrun.PY", line 45, in <module>
t.Push.find_all(conn, cursor)
File "c:\tradetools.py", line 198, in find_all
Push.find_streak(conn, cursor, name)
File "c:\tradetools.py", line 189, in find_strea
k
.format(c, name))
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\cursors.py", line 166, in execute
result = self._query(query)
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\cursors.py", line 322, in _query
conn.query(q)
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\connections.py", line 837, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\connections.py", line 1021, in _read_query_result
result.read()
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\connections.py", line 1304, in read
first_packet = self.connection._read_packet()
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\connections.py", line 983, in _read_packet
packet.check_error()
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\connections.py", line 395, in check_error
err.raise_mysql_exception(self._data)
File "C:\AppData\Local\Programs\Python\Python35\lib\site-packages\p
ymysql\err.py", line 102, in raise_mysql_exception
raise errorclass(errno, errval)
pymysql.err.ProgrammingError: (1064, "42000You have an error in your SQL syntax;
check the manual that corresponds to your MySQL server version for the right sy
ntax to use near 'AAPL''' at line 1")
c:\>
The code that its running through looks like this, why does the update add more commas to the name appl?
def find_streak(conn, cursor, name):
print(name)
cursor.execute("SELECT * FROM `trade_data`.`import_data`"
" WHERE name =%s;", name)
logs = cursor.fetchall()
cursor.execute("INSERT IGNORE INTO `trade_data`.`analysis`(`name`) "
"VALUES (%s) ON DUPLICATE KEY UPDATE "
"ndays=0;", name)
conn.commit()
logs = [list(x) for x in logs]
logs.sort()
....
cursor.execute ("UPDATE `trade_data`.`analysis` "
"SET `ndays` = {0} WHERE name='{1}'"
.format(c, name))
conn.commit()
its pulling from a table that looks like this
date|name|price|
and entering into a table that is
date|name|result