Develop Reference

Flask-Login trying to change my user.id into random strings instead of established ints?

Since updating my Flask app's dependencies, my Sentry is logging errors I don't understand. Changes to Flask-Login or Flask-Security-Too have thrown me off.
My user model has always used an Integer as its primary_key
class User(db.Model, UserMixin):
id = db.Column(db.Integer, primary_key=True)
I'm getting the following error:
(psycopg2.errors.InvalidTextRepresentation) invalid input syntax for integer: "5818f9aad20447b1b6a37c51f2c2xxxx"
LINE 3: WHERE "user".id = '5818f9aad20447b1b6a37c51f2c2xxxx'
Which I assume stems from Flask-Security-Too's datastore user lookup:
def find_user(self, **kwargs):
return self.user_model.query.filter_by(**kwargs).first()
But maybe the problem goes bath further to Flask-Login as it's getting this random ID from the session:
# Load user from Flask Session
user_id = session.get("_user_id")
if user_id is not None and self._user_callback is not None:
user = self._user_callback(user_id)
Is Flask-Security-Too's fs_uniquifier in my session? Is that what's stepping into the place of my integer ID? I'm hoping I can avoid refactoring my entire code and database and retain my user.id as an int. Any thoughts are appreciated.
Here is the full trace that has occurred 15 times since upgrading, all without halting users from accessing or submitting data:
InvalidTextRepresentation: invalid input syntax for integer: "63b93cd01b8f4e65a7daa83becc3xxxx"
LINE 3: WHERE "user".id = '63b93cd01b8f4e65a7daa83becc3xxxx'
^
File "sqlalchemy/engine/base.py", line 1819, in _execute_context
self.dialect.do_execute(
File "sqlalchemy/engine/default.py", line 732, in do_execute
cursor.execute(statement, parameters)
DataError: (psycopg2.errors.InvalidTextRepresentation) invalid input syntax for integer: "63b93cd01b8f4e65a7daa83becc3xxxx"
LINE 3: WHERE "user".id = '63b93cd01b8f4e65a7daa83becc3xxxx'
^
[SQL: SELECT "user".id AS user_id, "user".school_id AS user_school_id, "user".location_id AS user_location_id, "user".first_name AS user_first_name, "user".last_name AS user_last_name, "user".display_name AS user_display_name, "user".image AS user_image, "user".email AS user_email, "user".password AS use...
File "flask/app.py", line 2463, in wsgi_app
response = self.full_dispatch_request()
File "flask/app.py", line 1760, in full_dispatch_request
rv = self.handle_user_exception(e)
File "__init__.py", line 271, in error_router
return original_handler(e)
File "flask/app.py", line 1756, in full_dispatch_request
rv = self.preprocess_request()
File "flask/app.py", line 2247, in preprocess_request
rv = self.ensure_sync(before_func)()
File "flask_principal.py", line 477, in _on_before_request
identity = loader()
File "flask_security/core.py", line 245, in _identity_loader
if not isinstance(current_user._get_current_object(), AnonymousUserMixin):
File "werkzeug/local.py", line 516, in _get_current_object
return get_name(local()) # type: ignore
File "flask_login/utils.py", line 25, in <lambda>
current_user = LocalProxy(lambda: _get_user())
File "flask_login/utils.py", line 372, in _get_user
current_app.login_manager._load_user()
File "flask_login/login_manager.py", line 364, in _load_user
user = self._user_callback(user_id)
File "flask_security/core.py", line 221, in _user_loader
return _security.datastore.find_user(id=user_id)
File "flask_security/datastore.py", line 254, in find_user
return self.user_model.query.filter_by(**kwargs).first()
File "sqlalchemy/orm/query.py", line 2819, in first
return self.limit(1)._iter().first()
File "sqlalchemy/orm/query.py", line 2903, in _iter
result = self.session.execute(
File "sqlalchemy/orm/session.py", line 1712, in execute
result = conn._execute_20(statement, params or {}, execution_options)
File "sqlalchemy/engine/base.py", line 1631, in _execute_20
return meth(self, args_10style, kwargs_10style, execution_options)
File "sqlalchemy/sql/elements.py", line 332, in _execute_on_connection
return connection._execute_clauseelement(
File "sqlalchemy/engine/base.py", line 1498, in _execute_clauseelement
ret = self._execute_context(
File "sqlalchemy/engine/base.py", line 1862, in _execute_context
self._handle_dbapi_exception(
File "sqlalchemy/engine/base.py", line 2043, in _handle_dbapi_exception
util.raise_(
File "sqlalchemy/util/compat.py", line 208, in raise_
raise exception
File "sqlalchemy/engine/base.py", line 1819, in _execute_context
self.dialect.do_execute(
File "sqlalchemy/engine/default.py", line 732, in do_execute
cursor.execute(statement, parameters)
InFailedSqlTransaction: current transaction is aborted, commands ignored until end of transaction block
File "sqlalchemy/engine/base.py", line 1819, in _execute_context
self.dialect.do_execute(
File "sqlalchemy/engine/default.py", line 732, in do_execute
cursor.execute(statement, parameters)
InternalError: (psycopg2.errors.InFailedSqlTransaction) current transaction is aborted, commands ignored until end of transaction block
[SQL: SELECT "user".id AS user_id, "user".school_id AS user_school_id, "user".location_id AS user_location_id, "user".first_name AS user_first_name, "user".last_name AS user_last_name, "user".display_name AS user_display_name, "user".image AS user_image, "user".email AS user_email, "user".password AS user_password, "user".fs_uniquifier AS user_fs_uniquifier, "user".phone AS user_phone...
File "flask/app.py", line 2486, in __call__
return self.wsgi_app(environ, start_response)
File "flask/app.py", line 2466, in wsgi_app
response = self.handle_exception(e)
File "__init__.py", line 271, in error_router
return original_handler(e)
File "flask/app.py", line 1662, in handle_exception
server_error = self.ensure_sync(handler)(server_error)
File "__init__.py", line 21, in crash_page
return render_template('error_pages/500.html'), 500
File "flask/templating.py", line 147, in render_template
return _render(app, template, context)
File "flask/templating.py", line 128, in _render
app.update_template_context(context)
File "flask/app.py", line 932, in update_template_context
context.update(func())
File "flask_login/utils.py", line 407, in _user_context_processor
return dict(current_user=_get_user())
File "flask_login/utils.py", line 372, in _get_user
current_app.login_manager._load_user()
File "flask_login/login_manager.py", line 364, in _load_user
user = self._user_callback(user_id)
File "flask_security/core.py", line 221, in _user_loader
return _security.datastore.find_user(id=user_id)
File "flask_security/datastore.py", line 254, in find_user
return self.user_model.query.filter_by(**kwargs).first()
File "sqlalchemy/orm/query.py", line 2819, in first
return self.limit(1)._iter().first()
File "sqlalchemy/orm/query.py", line 2903, in _iter
result = self.session.execute(
File "sqlalchemy/orm/session.py", line 1712, in execute
result = conn._execute_20(statement, params or {}, execution_options)
File "sqlalchemy/engine/base.py", line 1631, in _execute_20
return meth(self, args_10style, kwargs_10style, execution_options)
File "sqlalchemy/sql/elements.py", line 332, in _execute_on_connection
return connection._execute_clauseelement(
File "sqlalchemy/engine/base.py", line 1498, in _execute_clauseelement
ret = self._execute_context(
File "sqlalchemy/engine/base.py", line 1862, in _execute_context
self._handle_dbapi_exception(
File "sqlalchemy/engine/base.py", line 2043, in _handle_dbapi_exception
util.raise_(
File "sqlalchemy/util/compat.py", line 208, in raise_
raise exception
File "sqlalchemy/engine/base.py", line 1819, in _execute_context
self.dialect.do_execute(
File "sqlalchemy/engine/default.py", line 732, in do_execute
cursor.execute(statement, parameters)

Some additional info - as of FS2 3.4.0 it no longer references user.id at all - so that can be whatever you want. FS2 overrides flask_login's UserMixin::get_id - and returns the value of fs_uniquifier. And yes - that is what is stored in the session cookie - and no - nothing in the session cookie is encrypted (it is signed).
So: 1) are you overriding get_id()?
2) any chance that since it seems to be 'working' that you have some post-processing - maybe in a signal or something and it is calling the find_user or some other DB query?

How to Push data from xlsx excel sheet to sqlalchemy database using Transpose and pandas

I am trying to push the excel file by using to_sql function and using data frame after transpose the dataframe...
and this is my code looks like
import pandas as pd
import os
import sqlalchemy
# MySQL Connection
MYSQL_USER = 'xxxxx'
MYSQL_PASSWORD = 'xxxxxxxx'
MYSQL_HOST_IP = '127.0.0.1'
MYSQL_PORT = 3306
MYSQL_DATABASE = 'xlsx_test_db'
# connect db
engine = sqlalchemy.create_engine('mysql+mysqlconnector://' + MYSQL_USER + ':' + MYSQL_PASSWORD + '#' + MYSQL_HOST_IP + ':' + str(
MYSQL_PORT) + '/' + MYSQL_DATABASE, echo=False)
engine.connect()
mydir = (os.getcwd()).replace('\\', '/') + '/'
raw_lte = pd.read_excel(r'' + mydir + 'MNM_Rotterdam_5_Daily_Details-20191216081027.xlsx', sheet_name='raw_4G')
dft = raw_lte.T
dft.columns = dft.iloc[0]
dft = dft.iloc[1:]
# reading and insert one file at a time
for file in os.listdir('.'):
# only process excels files
file_basename, extension = file.split('.')
if extension == 'xlsx':
dft.to_sql(file_basename.lower(), con=engine, if_exists='replace')
and this the error
Traceback
Traceback (most recent call last):
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1193, in _execute_context
context)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\engine\default.py", line 507, in do_execute
cursor.execute(statement, parameters)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\mysql\connector\cursor.py", line 551, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\mysql\connector\connection.py", line 490, in cmd_query
result = self._handle_result(self._send_cmd(ServerCmd.QUERY, query))
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\mysql\connector\connection.py", line 395, in _handle_result
raise errors.get_exception(packet)
mysql.connector.errors.ProgrammingError: 1170 (42000): BLOB/TEXT column 'index' used in key specification without a key length
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:/Users/DELL/PycharmProjects/automateDB/swap.py", line 36, in <module>
dft.to_sql(file_basename.lower(), con=engine, if_exists='replace')
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\pandas\core\generic.py", line 2532, in to_sql
dtype=dtype, method=method)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\pandas\io\sql.py", line 460, in to_sql
chunksize=chunksize, dtype=dtype, method=method)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\pandas\io\sql.py", line 1173, in to_sql
table.create()
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\pandas\io\sql.py", line 585, in create
self._execute_create()
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\pandas\io\sql.py", line 569, in _execute_create
self.table.create()
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\sql\schema.py", line 778, in create
checkfirst=checkfirst)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1940, in _run_visitor
conn._run_visitor(visitorcallable, element, **kwargs)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1549, in _run_visitor
**kwargs).traverse_single(element)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\sql\visitors.py", line 121, in traverse_single
return meth(obj, **kw)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\sql\ddl.py", line 796, in visit_table
self.traverse_single(index)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\sql\visitors.py", line 121, in traverse_single
return meth(obj, **kw)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\sql\ddl.py", line 823, in visit_index
self.connection.execute(CreateIndex(index))
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\engine\base.py", line 948, in execute
return meth(self, multiparams, params)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\sql\ddl.py", line 68, in _execute_on_connection
return connection._execute_ddl(self, multiparams, params)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1009, in _execute_ddl
compiled
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1200, in _execute_context
context)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1413, in _handle_dbapi_exception
exc_info
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\util\compat.py", line 203, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\util\compat.py", line 186, in reraise
raise value.with_traceback(tb)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1193, in _execute_context
context)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\sqlalchemy\engine\default.py", line 507, in do_execute
cursor.execute(statement, parameters)
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\mysql\connector\cursor.py", line 551, in execute
self._handle_result(self._connection.cmd_query(stmt))
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\mysql\connector\connection.py", line 490, in cmd_query
result = self._handle_result(self._send_cmd(ServerCmd.QUERY, query))
File "C:\Users\DELL\PycharmProjects\MyALLRefProf\venv\lib\site-packages\mysql\connector\connection.py", line 395, in _handle_result
raise errors.get_exception(packet)
sqlalchemy.exc.ProgrammingError: (mysql.connector.errors.ProgrammingError) 1170 (42000): BLOB/TEXT column 'index' used in key specification without a key length [SQL: 'CREATE INDEX `ix_mnm_rotterdam_5_daily_details-20191216081027_index` ON `mnm_rotterdam_5_daily_details-20191216081027` (`index`)'] (Background on this error at: http://sqlalche.me/e/f405)
I think there's a problem with the excel format, but I don't know to solve this issue
Note:
I tried to use this
raw_4G.to_sql(file_basename.lower(), con=engine, if_exists='replace')
insted of
dft.to_sql(file_basename.lower(), con=engine, if_exists='replace')
It works but it gives me error in the run time with this
Traceback (most recent call last):
File "C:/Users/DELL/PycharmProjects/automateDB/swap.py", line 34, in <module>
file_basename, extension = file.split('.')
ValueError: not enough values to unpack (expected 2, got 1)

sqlalchemy.exc.ResourceClosedError: This transaction is closed

Facing the following error in a Pyramid Application while using SQLAlchemy and Zope Transaction Manager.
This is how I am creating a scoped session:
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.schema import MetaData
from sqlalchemy.orm import (
scoped_session,
sessionmaker,
)
from zope.sqlalchemy import ZopeTransactionExtension
metadata = MetaData(naming_convention=NAMING_CONVENTION)
Base = declarative_base(metadata=metadata)
DBSession = scoped_session(sessionmaker(extension=ZopeTransactionExtension(keep_session=True)))
I am syncing my session creation and removal with my requests in the following way:
#view_config(route_name='social_get_individual_assets', renderer='json', effective_principals=2, permission='edit')
def social_get_individual_assets(requestJson):
session = DBSession()
try:
body = requestJson.request.json_body
search_term = body['text']
horizontal = body['horizontal']
vertical = body['vertical']
log.info("social get individual assets request: %s", str(search_term).encode(encoding='utf_8'))
json_data = get_individual_assets(session, search_term, horizontal, vertical)
log.info("social get assets response: %s", str(search_term).encode(encoding='utf_8'))
transaction.commit()
DBSession.remove()
return json_data
except Exception as e:
session.rollback()
log.exception(e)
raise e
For some reason, I keep running into this error all the time:
2017-12-06 13:32:07,965 ERROR [invideoapp.views.default:465] An operation previously failed, with traceback:
File "/usr/lib64/python3.5/threading.py", line 882, in _bootstrap
self._bootstrap_inner()
File "/usr/lib64/python3.5/threading.py", line 914, in _bootstrap_inner
self.run()
File "/usr/lib64/python3.5/threading.py", line 862, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.5/site-packages/waitress/task.py", line 78, in handler_thread
task.service()
File "/usr/local/lib/python3.5/site-packages/waitress/channel.py", line 338, in service
task.service()
File "/usr/local/lib/python3.5/site-packages/waitress/task.py", line 169, in service
self.execute()
File "/usr/local/lib/python3.5/site-packages/waitress/task.py", line 399, in execute
app_iter = self.channel.server.application(env, start_response)
File "/usr/local/lib/python3.5/site-packages/pyramid/router.py", line 270, in __call__
response = self.execution_policy(environ, self)
File "/usr/local/lib/python3.5/site-packages/pyramid_retry/__init__.py", line 114, in retry_policy
response = router.invoke_request(request)
File "/usr/local/lib/python3.5/site-packages/pyramid/router.py", line 249, in invoke_request
response = handle_request(request)
File "/usr/local/lib/python3.5/site-packages/pyramid_tm/__init__.py", line 136, in tm_tween
response = handler(request)
File "/usr/local/lib/python3.5/site-packages/pyramid/tweens.py", line 39, in excview_tween
response = handler(request)
File "/usr/local/lib/python3.5/site-packages/pyramid/router.py", line 156, in handle_request
view_name
File "/usr/local/lib/python3.5/site-packages/pyramid/view.py", line 642, in _call_view
response = view_callable(context, request)
File "/usr/local/lib/python3.5/site-packages/pyramid/viewderivers.py", line 439, in rendered_view
result = view(context, request)
File "/usr/local/lib/python3.5/site-packages/pyramid/viewderivers.py", line 148, in _requestonly_view
response = view(request)
File "/home/ttv/invideoapp/invideoapp/views/default.py", line 148, in upload_image
transaction.commit()
File "/usr/local/lib/python3.5/site-packages/transaction/_manager.py", line 131, in commit
return self.get().commit()
File "/usr/local/lib/python3.5/site-packages/transaction/_transaction.py", line 308, in commit
t, v, tb = self._saveAndGetCommitishError()
File "/usr/local/lib/python3.5/site-packages/transaction/_transaction.py", line 301, in commit
self._commitResources()
File "/usr/local/lib/python3.5/site-packages/transaction/_transaction.py", line 446, in _commitResources
reraise(t, v, tb)
File "/usr/local/lib/python3.5/site-packages/transaction/_compat.py", line 54, in reraise
raise value
File "/usr/local/lib/python3.5/site-packages/transaction/_transaction.py", line 423, in _commitResources
rm.tpc_vote(self)
File "/usr/local/lib/python3.5/site-packages/zope/sqlalchemy/datamanager.py", line 109, in tpc_vote
self.tx.commit()
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/session.py", line 459, in commit
self._assert_active(prepared_ok=True)
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/session.py", line 285, in _assert_active
raise sa_exc.ResourceClosedError(closed_msg)
sqlalchemy.exc.ResourceClosedError: This transaction is closed
Traceback (most recent call last):
File "/home/ttv/invideoapp/invideoapp/views/default.py", line 451, in update_master_json
user = getUser(session,user_id)
File "/home/ttv/invideoapp/invideoapp/invideomodules/auth_processing.py", line 12, in getUser
raise e
File "/home/ttv/invideoapp/invideoapp/invideomodules/auth_processing.py", line 8, in getUser
query = session.query(User).filter(User.user_id == userid).first()
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/query.py", line 2755, in first
ret = list(self[0:1])
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/query.py", line 2547, in __getitem__
return list(res)
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/query.py", line 2855, in __iter__
return self._execute_and_instances(context)
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/query.py", line 2876, in _execute_and_instances
close_with_result=True)
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/query.py", line 2885, in _get_bind_args
**kw
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/query.py", line 2867, in _connection_from_session
conn = self.session.connection(**kw)
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/session.py", line 966, in connection
execution_options=execution_options)
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/session.py", line 971, in _connection_for_bind
engine, execution_options)
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/session.py", line 417, in _connection_for_bind
self.session.dispatch.after_begin(self.session, self, conn)
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/event/attr.py", line 256, in __call__
fn(*args, **kw)
File "/usr/local/lib/python3.5/site-packages/zope/sqlalchemy/datamanager.py", line 237, in after_begin
join_transaction(session, self.initial_state, self.transaction_manager, self.keep_session)
File "/usr/local/lib/python3.5/site-packages/zope/sqlalchemy/datamanager.py", line 211, in join_transaction
DataManager(session, initial_state, transaction_manager, keep_session=keep_session)
File "/usr/local/lib/python3.5/site-packages/zope/sqlalchemy/datamanager.py", line 73, in __init__
transaction_manager.get().join(self)
File "/usr/local/lib/python3.5/site-packages/transaction/_transaction.py", line 179, in join
self._prior_operation_failed() # doesn't return
File "/usr/local/lib/python3.5/site-packages/transaction/_transaction.py", line 173, in _prior_operation_failed
self._failure_traceback.getvalue())
transaction.interfaces.TransactionFailedError: An operation previously failed, with traceback:
File "/usr/lib64/python3.5/threading.py", line 882, in _bootstrap
self._bootstrap_inner()
File "/usr/lib64/python3.5/threading.py", line 914, in _bootstrap_inner
self.run()
File "/usr/lib64/python3.5/threading.py", line 862, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.5/site-packages/waitress/task.py", line 78, in handler_thread
task.service()
File "/usr/local/lib/python3.5/site-packages/waitress/channel.py", line 338, in service
task.service()
File "/usr/local/lib/python3.5/site-packages/waitress/task.py", line 169, in service
self.execute()
File "/usr/local/lib/python3.5/site-packages/waitress/task.py", line 399, in execute
app_iter = self.channel.server.application(env, start_response)
File "/usr/local/lib/python3.5/site-packages/pyramid/router.py", line 270, in __call__
response = self.execution_policy(environ, self)
File "/usr/local/lib/python3.5/site-packages/pyramid_retry/__init__.py", line 114, in retry_policy
response = router.invoke_request(request)
File "/usr/local/lib/python3.5/site-packages/pyramid/router.py", line 249, in invoke_request
response = handle_request(request)
File "/usr/local/lib/python3.5/site-packages/pyramid_tm/__init__.py", line 136, in tm_tween
response = handler(request)
File "/usr/local/lib/python3.5/site-packages/pyramid/tweens.py", line 39, in excview_tween
response = handler(request)
File "/usr/local/lib/python3.5/site-packages/pyramid/router.py", line 156, in handle_request
view_name
File "/usr/local/lib/python3.5/site-packages/pyramid/view.py", line 642, in _call_view
response = view_callable(context, request)
File "/usr/local/lib/python3.5/site-packages/pyramid/viewderivers.py", line 439, in rendered_view
result = view(context, request)
File "/usr/local/lib/python3.5/site-packages/pyramid/viewderivers.py", line 148, in _requestonly_view
response = view(request)
File "/home/ttv/invideoapp/invideoapp/views/default.py", line 148, in upload_image
transaction.commit()
File "/usr/local/lib/python3.5/site-packages/transaction/_manager.py", line 131, in commit
return self.get().commit()
File "/usr/local/lib/python3.5/site-packages/transaction/_transaction.py", line 308, in commit
t, v, tb = self._saveAndGetCommitishError()
File "/usr/local/lib/python3.5/site-packages/transaction/_transaction.py", line 301, in commit
self._commitResources()
File "/usr/local/lib/python3.5/site-packages/transaction/_transaction.py", line 446, in _commitResources
reraise(t, v, tb)
File "/usr/local/lib/python3.5/site-packages/transaction/_compat.py", line 54, in reraise
raise value
File "/usr/local/lib/python3.5/site-packages/transaction/_transaction.py", line 423, in _commitResources
rm.tpc_vote(self)
File "/usr/local/lib/python3.5/site-packages/zope/sqlalchemy/datamanager.py", line 109, in tpc_vote
self.tx.commit()
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/session.py", line 459, in commit
self._assert_active(prepared_ok=True)
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/session.py", line 285, in _assert_active
raise sa_exc.ResourceClosedError(closed_msg)
sqlalchemy.exc.ResourceClosedError: This transaction is closed
Once I get this error, none of my Database connection stuff works. Only way out is to restart the entire application to make things work. Any idea why this is happening?

I had a similar problem, and I found using DBSession.begin_nested() solved the issue. So something like:
#view_config(route_name='social_get_individual_assets', renderer='json', effective_principals=2, permission='edit')
def social_get_individual_assets(requestJson):
try:
DBSession.begin_nested()
body = requestJson.request.json_body
search_term = body['text']
horizontal = body['horizontal']
vertical = body['vertical']
log.info("social get individual assets request: %s", str(search_term).encode(encoding='utf_8'))
json_data = get_individual_assets(session, search_term, horizontal, vertical)
log.info("social get assets response: %s", str(search_term).encode(encoding='utf_8'))
return json_data
except Exception as e:
DBSession.rollback()
log.exception(e)
raise e

Unknown PG numeric type 25

I am working on a project with Python-Flask, SQL Alchemy, & Postgres qith psycopg2 package.
The project's save the data in database successfully, but throws error "Unknown PG numeric type 25" while refreshing object.
project.second_payment_percent = int(request.form.get("second_payment_percent", 0)) if request.form.get("second_payment_percent", 0) else 0
project.second_payment_before = int(request.form.get("second_payment_before", 120))
project.currency_id = int(request.form.get("currency_code"))
project.security_deposit = int(request.form.get("security_deposit", 0))
project.apa = int(request.form.get("apa", 0))
project.vat_schema_id = request.form.get("vat_schema") if request.form.get("vat_schema", 0) else None
if project.apa > 0:
project.security_deposit = int(project.project_price) * int(project.apa) / 100
db.session.add(project)
db.session.commit()
db.session.refresh(project)
db.session.flush()
The error I get is
File "<project directory>utils/app_views.py", line 56, in decorated
return f(*args, **kwargs)
File "<project directory>views/admin/project.py", line 347, in admin_products_save_accommodation
db.session.refresh(project)
File "<project directory>models/project.py", line 297, in serialize
reviews, avg_rating = self.reviews
File "<project directory>models/project.py", line 269, in reviews
robj = db.session.query(ProjectReviews).filter_by(product_id=self.id).order_by('time_created').all()
File "<venv path>lib/python2.7/site-packages/sqlalchemy/orm/attributes.py", line 237, in __get__
return self.impl.get(instance_state(instance), dict_)
File "<venv path>lib/python2.7/site-packages/sqlalchemy/orm/attributes.py", line 578, in get
value = state._load_expired(state, passive)
File "<venv path>lib/python2.7/site-packages/sqlalchemy/orm/state.py", line 474, in _load_expired
self.manager.deferred_scalar_loader(self, toload)
File "<venv path>lib/python2.7/site-packages/sqlalchemy/orm/loading.py", line 664, in load_scalar_attributes
only_load_props=attribute_names)
File "<venv path>lib/python2.7/site-packages/sqlalchemy/orm/loading.py", line 219, in load_on_ident
return q.one()
File "<venv path>lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2693, in one
ret = list(self)
File "<venv path>lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2736, in __iter__
return self._execute_and_instances(context)
File "<venv path>lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2751, in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
File "<venv path>lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 914, in execute
return meth(self, multiparams, params)
File "<venv path>lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 323, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "<venv path>lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1010, in _execute_clauseelement
compiled_sql, distilled_params
File "<venv path>lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1161, in _execute_context
result = context.get_result_proxy()
File "<venv path>lib/python2.7/site-packages/sqlalchemy/dialects/postgresql/psycopg2.py", line 474, in get_result_proxy
return _result.ResultProxy(self)
File "<venv path>lib/python2.7/site-packages/sqlalchemy/engine/result.py", line 495, in __init__
self._init_metadata()
File "<venv path>lib/python2.7/site-packages/sqlalchemy/engine/result.py", line 524, in _init_metadata
self._metadata = ResultMetaData(self, metadata)
File "<venv path>lib/python2.7/site-packages/sqlalchemy/engine/result.py", line 221, in __init__
in enumerate(result_columns)
File "<venv path>lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 764, in get_result_processor
return type_._cached_result_processor(self.dialect, coltype)
File "<venv path>lib/python2.7/site-packages/sqlalchemy/sql/type_api.py", line 392, in _cached_result_processor
d[coltype] = rp = d['impl'].result_processor(dialect, coltype)
File "<venv path>lib/python2.7/site-packages/sqlalchemy/dialects/postgresql/psycopg2.py", line 361, in result_processor
"Unknown PG numeric type: %d" % coltype)
InvalidRequestError: Unknown PG numeric type: 25
I have tried many things, but failed.

I guess one of the fields in your table is a text and you are trying to retrieve it as a number.
postgres=# select * from pg_type where oid = 25;
-[ RECORD 1 ]--+---------
typname | text

SQL Alchemy : cannot connect using engine.connect()

I am using sqlalchemy and pandas to connect to db and create the dataframe from sql table.
This is my code:
port=3306
unix_socket='/var/lib/mysql/mysql.sock'
conn = "mysql://user:pwd#localhost/dbname?port=3306?unix_socket=/var/lib/mysql/mysql.sock"
_engine = create_engine(conn)
print conn
print _engine
db_conn=_engine.connect()
Output:
Traceback (most recent call last):
File "runner_train_test.py", line 37, in <module>
_engine.connect()
File "/usr/local/anaconda2/lib/python2.7/site-packages/sqlalchemy/engine/base. py", line 2018, in connect
return self._connection_cls(self, **kwargs)
File "/usr/local/anaconda2/lib/python2.7/site-packages/sqlalchemy/engine/base. py", line 72, in __init__
if connection is not None else engine.raw_connection()
File "/usr/local/anaconda2/lib/python2.7/site-packages/sqlalchemy/engine/base. py", line 2104, in raw_connection
self.pool.unique_connection, _connection)
File "/usr/local/anaconda2/lib/python2.7/site-packages/sqlalchemy/engine/base. py", line 2074, in _wrap_pool_connect
return fn()
File "/usr/local/anaconda2/lib/python2.7/site-packages/sqlalchemy/pool.py", li ne 318, in unique_connection
return _ConnectionFairy._checkout(self)
File "/usr/local/anaconda2/lib/python2.7/site-packages/sqlalchemy/pool.py", li ne 713, in _checkout
fairy = _ConnectionRecord.checkout(pool)
File "/usr/local/anaconda2/lib/python2.7/site-packages/sqlalchemy/pool.py", li ne 480, in checkout
rec = pool._do_get()
File "/usr/local/anaconda2/lib/python2.7/site-packages/sqlalchemy/pool.py", li ne 1060, in _do_get
self._dec_overflow()
File "/usr/local/anaconda2/lib/python2.7/site-packages/sqlalchemy/util/langhel pers.py", line 60, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/usr/local/anaconda2/lib/python2.7/site-packages/sqlalchemy/pool.py", li ne 1057, in _do_get
return self._create_connection()
File "/usr/local/anaconda2/lib/python2.7/site-packages/sqlalchemy/pool.py", li ne 323, in _create_connection
return _ConnectionRecord(self)
File "/usr/local/anaconda2/lib/python2.7/site-packages/sqlalchemy/pool.py", li ne 449, in __init__
self.connection = self.__connect()
File "/usr/local/anaconda2/lib/python2.7/site-packages/sqlalchemy/pool.py", li ne 607, in __connect
connection = self.__pool._invoke_creator(self)
File "/usr/local/anaconda2/lib/python2.7/site-packages/sqlalchemy/engine/strat egies.py", line 97, in connect
return dialect.connect(*cargs, **cparams)
File "/usr/local/anaconda2/lib/python2.7/site-packages/sqlalchemy/engine/defau lt.py", line 385, in connect
return self.dbapi.connect(*cargs, **cparams)
File "/usr/local/anaconda2/lib/python2.7/site-packages/MySQLdb/__init__.py", l ine 81, in Connect
return Connection(*args, **kwargs)
File "/usr/local/anaconda2/lib/python2.7/site-packages/MySQLdb/connections.py" , line 193, in __init__
super(Connection, self).__init__(*args, **kwargs2)
TypeError: an integer is required

Your connection string looks wrong. The sqlalchemy documentation states the following string: mysql+mysqldb://<user>:<password>#<host>[:<port>]/<dbname> or an example for google cloud: mysql+mysqldb://root#/<dbname>?unix_socket=/cloudsql/<projectid>:<instancename> so your string should probabably be:
mysql://user#/dbname?unix_socket=/var/lib/mysql/mysql.sock