Related
I am trying to access a table in an AWS bucket. When I try to access it using awswrangler.read_parquet function I get an error saying that I am not able to access that file because I can't create new threads. I am usually able to access that file after waiting 30min+, but that doesn't tell me how to solve the problem. Here are more details about the command:
aws_df = wr.s3.read_parquet(path=self._filepath, **self._load_args)
File "/home/ec2-user/anaconda3/lib/python3.7/site-packages/awswrangler/s3/_read_parquet.py", line 721, in read_parquet
read_func=_read_parquet, paths=paths, version_ids=versions, use_threads=use_threads, kwargs=args
File "/home/ec2-user/anaconda3/lib/python3.7/site-packages/awswrangler/s3/_read.py", line 145, in _read_dfs_from_multiple_paths
return list(df for df in executor.map(partial_read_func, paths, versions))
File "/home/ec2-user/anaconda3/lib/python3.7/site-packages/awswrangler/s3/_read.py", line 145, in <genexpr>
return list(df for df in executor.map(partial_read_func, paths, versions))
File "/home/ec2-user/anaconda3/lib/python3.7/concurrent/futures/_base.py", line 586, in result_iterator
yield fs.pop().result()
File "/home/ec2-user/anaconda3/lib/python3.7/concurrent/futures/_base.py", line 432, in result
return self.__get_result()
File "/home/ec2-user/anaconda3/lib/python3.7/concurrent/futures/_base.py", line 384, in __get_result
raise self._exception
File "/home/ec2-user/anaconda3/lib/python3.7/concurrent/futures/thread.py", line 57, in run
result = self.fn(*self.args, **self.kwargs)
File "/home/ec2-user/anaconda3/lib/python3.7/site-packages/awswrangler/s3/_read_parquet.py", line 495, in _read_parquet
version_id=version_id,
File "/home/ec2-user/anaconda3/lib/python3.7/site-packages/awswrangler/s3/_read_parquet.py", line 440, in _read_parquet_file
source=f, read_dictionary=categories
File "/home/ec2-user/anaconda3/lib/python3.7/site-packages/awswrangler/s3/_read_parquet.py", line 40, in _pyarrow_parquet_file_wrapper
return pyarrow.parquet.ParquetFile(source=source, read_dictionary=read_dictionary)
File "/home/ec2-user/anaconda3/lib/python3.7/site-packages/pyarrow/parquet.py", line 201, in __init__
read_dictionary=read_dictionary, metadata=metadata)
File "pyarrow/_parquet.pyx", line 1021, in pyarrow._parquet.ParquetReader.open
File "/home/ec2-user/anaconda3/lib/python3.7/site-packages/awswrangler/s3/_fs.py", line 569, in read
self._fetch(self._loc, self._loc + length)
File "/home/ec2-user/anaconda3/lib/python3.7/site-packages/awswrangler/s3/_fs.py", line 376, in _fetch
self._cache = self._fetch_range_proxy(self._start, self._end)
File "/home/ec2-user/anaconda3/lib/python3.7/site-packages/awswrangler/s3/_fs.py", line 359, in _fetch_range_proxy
itertools.repeat(self._version_id),
File "/home/ec2-user/anaconda3/lib/python3.7/concurrent/futures/_base.py", line 575, in map
fs = [self.submit(fn, *args) for args in zip(*iterables)]
File "/home/ec2-user/anaconda3/lib/python3.7/concurrent/futures/_base.py", line 575, in <listcomp>
fs = [self.submit(fn, *args) for args in zip(*iterables)]
File "/home/ec2-user/anaconda3/lib/python3.7/concurrent/futures/thread.py", line 160, in submit
self._adjust_thread_count()
File "/home/ec2-user/anaconda3/lib/python3.7/concurrent/futures/thread.py", line 181, in _adjust_thread_count
t.start()
File "/home/ec2-user/anaconda3/lib/python3.7/threading.py", line 847, in start
_start_new_thread(self._bootstrap, ())
RuntimeError: can't start new thread
I was able to solve the problem by adding a sleep between my commands.
import time
time.sleep(10)
I have a Flask based backend application and when running the code locally, this error does not happen, but it does happen when deployed in my server in a kubernetes pod.
Interestingly enough, when you run the code for the first time, it fails instantly (when running the Future) but when you run it again the error still happens, but the code continues (and completes successfully). Pretty bizarre.
The error I am getting is:
2021-07-30 14:59:23,537 - mit_backend.logic.optimize - INFO - Inside first pass run code
2021-07-30 14:59:23,587 - mit_backend.logic.optimize - INFO - Optimising region ML10F
2021-07-30 14:59:23,588 - mit_backend.logic.optimize - INFO - Optimising region ML47F
2021-07-30 14:59:23,589 - mit_backend.logic.optimize - INFO - Optimising region ML40F
--- Logging error ---
concurrent.futures.process._RemoteTraceback:
"""
Traceback (most recent call last):
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1277, in _execute_context
cursor, statement, parameters, context
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute
cursor.execute(statement, parameters)
psycopg2.OperationalError: SSL error: wrong version number
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/usr/lib64/python3.6/concurrent/futures/process.py", line 175, in _process_worker
r = call_item.fn(*call_item.args, **call_item.kwargs)
File "/code/mit_backend/logic/optimize.py", line 241, in optimize_first_pass_run
config = get_csv_config(user_identity)
File "/code/mit_backend/modules/v1/__init__.py", line 623, in get_csv_config
res = CSVConfig.query.filter(CSVConfig.user_id == user_identity).first()
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/query.py", line 3429, in first
ret = list(self[0:1])
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/query.py", line 3203, in __getitem__
return list(res)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/query.py", line 3535, in __iter__
return self._execute_and_instances(context)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/query.py", line 3560, in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1011, in execute
return meth(self, multiparams, params)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1130, in _execute_clauseelement
distilled_params,
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1317, in _execute_context
e, statement, parameters, cursor, context
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1511, in _handle_dbapi_exception
sqlalchemy_exception, with_traceback=exc_info[2], from_=e
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 182, in raise_
raise exception
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1277, in _execute_context
cursor, statement, parameters, context
File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) SSL error: wrong version number
[SQL: SELECT csv_config.user_id AS csv_config_user_id, csv_config."csvHeaders" AS "csv_config_csvHeaders", csv_config."deliveryName" AS "csv_config_deliveryName", csv_config.id AS csv_config_id, csv_config.line1 AS csv_config_line1, csv_config.line2 AS csv_config_line2, csv_config.quantity AS csv_config_quantity, csv_config."routeNumber" AS "csv_config_routeNumber", csv_config.suburb AS csv_config_suburb, csv_config.weight AS csv_config_weight
FROM csv_config
WHERE csv_config.user_id = %(user_id_1)s
LIMIT %(param_1)s]
[parameters: {'user_id_1': 'sftp', 'param_1': 1}]
(Background on this error at: http://sqlalche.me/e/13/e3q8)
"""
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/code/mit_backend/logic/optimize.py", line 300, in first_pass_run
future_df_result = future.result()
File "/usr/lib64/python3.6/concurrent/futures/_base.py", line 425, in result
return self.__get_result()
File "/usr/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result
raise self._exception
sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) SSL error: wrong version number
[SQL: SELECT csv_config.user_id AS csv_config_user_id, csv_config."csvHeaders" AS "csv_config_csvHeaders", csv_config."deliveryName" AS "csv_config_deliveryName", csv_config.id AS csv_config_id, csv_config.line1 AS csv_config_line1, csv_config.line2 AS csv_config_line2, csv_config.quantity AS csv_config_quantity, csv_config."routeNumber" AS "csv_config_routeNumber", csv_config.suburb AS csv_config_suburb, csv_config.weight AS csv_config_weight
FROM csv_config
WHERE csv_config.user_id = %(user_id_1)s
LIMIT %(param_1)s]
[parameters: {'user_id_1': 'sftp', 'param_1': 1}]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib64/python3.6/logging/__init__.py", line 994, in emit
msg = self.format(record)
File "/usr/lib64/python3.6/logging/__init__.py", line 840, in format
return fmt.format(record)
File "/usr/lib64/python3.6/logging/__init__.py", line 577, in format
record.message = record.getMessage()
File "/usr/lib64/python3.6/logging/__init__.py", line 338, in getMessage
msg = msg % self.args
TypeError: not all arguments converted during string formatting
Call stack:
File "/usr/lib64/python3.6/threading.py", line 884, in _bootstrap
self._bootstrap_inner()
File "/usr/lib64/python3.6/threading.py", line 916, in _bootstrap_inner
self.run()
File "/usr/lib64/python3.6/threading.py", line 864, in run
self._target(*self._args, **self._kwargs)
File "/usr/lib64/python3.6/socketserver.py", line 654, in process_request_thread
self.finish_request(request, client_address)
File "/usr/lib64/python3.6/socketserver.py", line 364, in finish_request
self.RequestHandlerClass(request, client_address, self)
File "/usr/lib64/python3.6/socketserver.py", line 724, in __init__
self.handle()
File "/usr/local/lib/python3.6/site-packages/werkzeug/serving.py", line 345, in handle
BaseHTTPRequestHandler.handle(self)
File "/usr/lib64/python3.6/http/server.py", line 418, in handle
self.handle_one_request()
File "/usr/local/lib/python3.6/site-packages/werkzeug/serving.py", line 379, in handle_one_request
return self.run_wsgi()
File "/usr/local/lib/python3.6/site-packages/werkzeug/serving.py", line 323, in run_wsgi
execute(self.server.app)
File "/usr/local/lib/python3.6/site-packages/werkzeug/serving.py", line 312, in execute
application_iter = app(environ, start_response)
File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 2464, in __call__
return self.wsgi_app(environ, start_response)
File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 2447, in wsgi_app
response = self.full_dispatch_request()
File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1950, in full_dispatch_request
rv = self.dispatch_request()
File "/usr/local/lib/python3.6/site-packages/flask/app.py", line 1936, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/usr/local/lib/python3.6/site-packages/flask_restx/api.py", line 375, in wrapper
resp = resource(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/flask/views.py", line 89, in view
return self.dispatch_request(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/flask_restx/resource.py", line 44, in dispatch_request
resp = meth(*args, **kwargs)
File "/code/mit_backend/modules/v1/controllers/manifest.py", line 161, in post
files, df, sftp_date_latest = do_optimize(addresses, user_identity)
File "/code/mit_backend/logic/optimize.py", line 38, in do_optimize
df = first_pass_run(df, user_identity)
File "/code/mit_backend/logic/optimize.py", line 307, in first_pass_run
LOG.error("Exception is ", e.__cause__)
Message: 'Exception is '
Arguments: (_RemoteTraceback('\n"""\nTraceback (most recent call last):\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1277, in _execute_context\n cursor, statement, parameters, context\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute\n cursor.execute(statement, parameters)\npsycopg2.OperationalError: SSL error: decryption failed or bad record mac\n\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File "/usr/lib64/python3.6/concurrent/futures/process.py", line 175, in _process_worker\n r = call_item.fn(*call_item.args, **call_item.kwargs)\n File "/code/mit_backend/logic/optimize.py", line 241, in optimize_first_pass_run\n config = get_csv_config(user_identity)\n File "/code/mit_backend/modules/v1/__init__.py", line 623, in get_csv_config\n res = CSVConfig.query.filter(CSVConfig.user_id == user_identity).first()\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/query.py", line 3429, in first\n ret = list(self[0:1])\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/query.py", line 3203, in __getitem__\n return list(res)\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/query.py", line 3535, in __iter__\n return self._execute_and_instances(context)\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/orm/query.py", line 3560, in _execute_and_instances\n result = conn.execute(querycontext.statement, self._params)\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1011, in execute\n return meth(self, multiparams, params)\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/sql/elements.py", line 298, in _execute_on_connection\n return connection._execute_clauseelement(self, multiparams, params)\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1130, in _execute_clauseelement\n distilled_params,\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1317, in _execute_context\n e, statement, parameters, cursor, context\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1511, in _handle_dbapi_exception\n sqlalchemy_exception, with_traceback=exc_info[2], from_=e\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/util/compat.py", line 182, in raise_\n raise exception\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/base.py", line 1277, in _execute_context\n cursor, statement, parameters, context\n File "/usr/local/lib64/python3.6/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute\n cursor.execute(statement, parameters)\nsqlalchemy.exc.OperationalError: (psycopg2.OperationalError) SSL error: decryption failed or bad record mac\n\n[SQL: SELECT csv_config.user_id AS csv_config_user_id, csv_config."csvHeaders" AS "csv_config_csvHeaders", csv_config."deliveryName" AS "csv_config_deliveryName", csv_config.id AS csv_config_id, csv_config.line1 AS csv_config_line1, csv_config.line2 AS csv_config_line2, csv_config.quantity AS csv_config_quantity, csv_config."routeNumber" AS "csv_config_routeNumber", csv_config.suburb AS csv_config_suburb, csv_config.weight AS csv_config_weight \nFROM csv_config \nWHERE csv_config.user_id = %(user_id_1)s \n LIMIT %(param_1)s]\n[parameters: {\'user_id_1\': \'sftp\', \'param_1\': 1}]\n(Background on this error at: http://sqlalche.me/e/13/e3q8)\n"""',),)
The error happens inside a futures ProcessPoolExecutor:
def first_pass_run(df, user_identity):
LOG.info("Inside first pass run code")
first_json = True
df_final = None
df_grouped = df.groupby('Route Number')
auth_claim = get_authorization_claims_from_header()
with concurrent.futures.ProcessPoolExecutor(max_workers=8) as executor:
_futures = []
for region_name, _df in df_grouped:
_futures.append(
executor.submit(
optimize_first_pass_run, region_name, _df, user_identity, auth_claim
))
for future in concurrent.futures.as_completed(_futures):
try:
future_df_result = future.result()
if first_json:
df_final = future_df_result
first_json = False
else:
df_final = df_final.append(future_df_result, ignore_index=True)
except Exception as e:
LOG.error("Exception is ", e.__cause__)
I start my SQLAlchemy engine in a pessimistic fashion:
pg_db = SQLAlchemy(app)
engine = create_engine(app.config['SQLALCHEMY_DATABASE_URI'], pool_pre_ping=True)
The code is not running on WSGI or gunicorn, it's just as simple as:
if __name__ == '__main__':
LOG.info('running environment: %s', os.environ.get('ENV', 'production'))
app.config['DEBUG'] = os.environ.get('ENV') == 'development'
app.run(host='0.0.0.0', port=5001, debug=False, threaded=True)
Any clues why this might be happening? Thank you.
I haven't been able to find a proper fix for this, but I implemented the following things that although the exception is still being thrown, the code retries and ends up succeeding:
Removing the connection pool
engine = create_engine(app.config['SQLALCHEMY_DATABASE_URI'], poolclass=NullPool)
Event listeners
#event.listens_for(engine, "connect")
def connect(dbapi_connection, connection_record):
connection_record.info['pid'] = os.getpid()
#event.listens_for(engine, "checkout")
def checkout(dbapi_connection, connection_record, connection_proxy):
pid = os.getpid()
if connection_record.info['pid'] != pid:
connection_record.connection = connection_proxy.connection = None
raise exc.DisconnectionError(
"Connection record belongs to pid %s, "
"attempting to check out in pid %s" %
(connection_record.info['pid'], pid)
)
but, as per the documentation, the two definitions above relate to an optimistic way of handling the connection and using Pools
Reconnecting before initializing the Process Pool
engine.dispose()
with concurrent.futures.ProcessPoolExecutor(max_workers=8) as executor:
_futures = []
for region_name, _df in df_grouped:
_futures.append(
executor.submit(
optimize_first_pass_run, region_name, _df, user_identity, auth_claim, True
))
for future in concurrent.futures.as_completed(_futures):
future_df_result = future.result()
Adding retries when trying to insert objects in the database
def add_new_record(row):
attempts = 0
while attempts <= 5:
attempts = attempts + 1
try:
pg_db.session.add(row)
pg_db.session.commit()
return True
except (SQLAlchemyError, psycopg2.OperationalError, sqlalchemy.exc.OperationalError) as e:
if attempts < 5:
LOG.warn("Attempt failed. Trying rollback")
pg_db.session.rollback()
LOG.warn(e.__cause__)
LOG.warn(e.__str__())
time.sleep(5)
else:
LOG.error("Maximum number of retries reached. Raising an error")
raise e
return False
I'm using TestLink-API-Python-client to work with Testlink application API. So, I need to get all testcases for the TestSuite with execution_type = 1 or execution_type = 2 to calculate the quantity of automated and manual tests (when execution_type = 1 it's manual tests, when execution_type = 2, it's automated tests).
For this purpose I used next method from the library getTestCasesForTestSuite(testsuite["id"], True, "full", getkeywords=True) as the result I got issue:
Traceback (most recent call last):
File "C:/Users/user1/PycharmProjects/script_for_testlink/script_for_testlink.py", line 871, in <module>
result = getAutomatedTestcasesForProject(TESTLINK_API_URL, TESTLINK_API_DEVKEY, project_in_testlink)
File "C:/Users/user1/PycharmProjects/script_for_testlink/script_for_testlink.py", line 112, in getAutomatedTestcasesForProject
testcases_for_testsuites = testlink.getTestCasesForTestSuite(testsuite["id"], True, "full", getkeywords=True)
File "C:\Users\user1\TAF\venv\lib\site-packages\testlink\testlinkdecorators.py", line 140, in wrapperReplaceTLResponseError
response = methodAPI(self, *argsPositional, **argsOptional)
File "C:\Users\user1\TAF\venv\lib\site-packages\testlink\testlinkdecorators.py", line 112, in wrapperAddDevKey
return methodAPI(self, *argsPositional, **argsOptional)
File "C:\Users\user1\TAF\venv\lib\site-packages\testlink\testlinkdecorators.py", line 100, in wrapperWithArgs
*argsPositional, **argsOptional)
File "C:\Users\user1\TAF\venv\lib\site-packages\testlink\testlinkapigeneric.py", line 1517, in callServerWithPosArgs
response = self._callServer(methodNameAPI, argsOptional)
File "C:\Users\user1\TAF\venv\lib\site-packages\testlink\testlinkapigeneric.py", line 1992, in _callServer
response = getattr(self.server.tl, methodNameAPI)(argsAPI)
File "c:\users\user1\appdata\local\programs\python\python37\Lib\xmlrpc\client.py", line 1117, in __call__
return self.__send(self.__name, args)
File "c:\users\user1\appdata\local\programs\python\python37\Lib\xmlrpc\client.py", line 1457, in __request
verbose=self.__verbose
File "c:\users\user1\appdata\local\programs\python\python37\Lib\xmlrpc\client.py", line 1159, in request
return self.single_request(host, handler, request_body, verbose)
File "c:\users\user1\appdata\local\programs\python\python37\Lib\xmlrpc\client.py", line 1175, in single_request
return self.parse_response(resp)
File "c:\users\user1\appdata\local\programs\python\python37\Lib\xmlrpc\client.py", line 1345, in parse_response
p.close()
File "c:\users\user1\appdata\local\programs\python\python37\Lib\xmlrpc\client.py", line 453, in close
parser.Parse(b"", True) # end of data
xml.parsers.expat.ExpatError: not well-formed (invalid token): line 5186, column 0
How can I get testcases with field execution_type without issues?
Having an issue with the api for Odoo v13. I am able to get the server info but for some reason the uid is not being returned
import xmlrpc.client
url ="localhost:8069"
db = "pnv3"
username = "test"
password = "test"
common = xmlrpc.client.ServerProxy('{}/xmlrpc/2/common'.format(url))
print(common.version())
uid = common.authenticate(db, username, password, url)
print(uid)
getting this error
Traceback (most recent call last):
File "C:/Users/Web Content/.PyCharmCE2019.3/config/scratches/scratch.py", line 11, in <module>
uid = common.authenticate(db, username, password, url)
File "C:\Users\Web Content\AppData\Local\Programs\Python\Python37\lib\xmlrpc\client.py", line 1112, in __call__
return self.__send(self.__name, args)
File "C:\Users\Web Content\AppData\Local\Programs\Python\Python37\lib\xmlrpc\client.py", line 1452, in __request
verbose=self.__verbose
File "C:\Users\Web Content\AppData\Local\Programs\Python\Python37\lib\xmlrpc\client.py", line 1154, in request
return self.single_request(host, handler, request_body, verbose)
File "C:\Users\Web Content\AppData\Local\Programs\Python\Python37\lib\xmlrpc\client.py", line 1170, in single_request
return self.parse_response(resp)
File "C:\Users\Web Content\AppData\Local\Programs\Python\Python37\lib\xmlrpc\client.py", line 1342, in parse_response
return u.close()
File "C:\Users\Web Content\AppData\Local\Programs\Python\Python37\lib\xmlrpc\client.py", line 656, in close
raise Fault(**self._stack[0])
xmlrpc.client.Fault: <Fault 1: 'Traceback (most recent call last):\n File "/odoo/odoo-server/odoo/modules/registry.py", line 59, in __new__\n return cls.registries[db_name]\n File "/odoo/odoo-server/odoo/tools/func.py", line 69, in wrapper\n return func(self, *args, **kwargs)\n File "/odoo/odoo-server/odoo/tools/lru.py", line 44, in __getitem__\n a = self.d[obj].me\nKeyError: \'pnv3\'\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File "/odoo/odoo-server/odoo/addons/base/controllers/rpc.py", line 63, in xmlrpc_2\n response = self._xmlrpc(service)\n File "/odoo/odoo-server/odoo/addons/base/controllers/rpc.py", line 43, in _xmlrpc\n result = dispatch_rpc(service, method, params)\n File "/odoo/odoo-server/odoo/http.py", line 138, in dispatch_rpc\n result = dispatch(method, params)\n File "/odoo/odoo-server/odoo/service/common.py", line 61, in dispatch\n return g[exp_method_name](*params)\n File "/odoo/odoo-server/odoo/service/common.py", line 30, in exp_authenticate\n res_users = odoo.registry(db)[\'res.users\']\n File "/odoo/odoo-server/odoo/__init__.py", line 104, in registry\n return modules.registry.Registry(database_name)\n File "/odoo/odoo-server/odoo/modules/registry.py", line 61, in __new__\n return cls.new(db_name)\n File "/odoo/odoo-server/odoo/modules/registry.py", line 73, in new\n registry.init(db_name)\n File "/odoo/odoo-server/odoo/modules/registry.py", line 141, in init\n with closing(self.cursor()) as cr:\n File "/odoo/odoo-server/odoo/modules/registry.py", line 492, in cursor\n return self._db.cursor()\n File "/odoo/odoo-server/odoo/sql_db.py", line 649, in cursor\n return Cursor(self.__pool, self.dbname, self.dsn, serialized=serialized)\n File "/odoo/odoo-server/odoo/sql_db.py", line 186, in __init__\n self._cnx = pool.borrow(dsn)\n File "/odoo/odoo-server/odoo/sql_db.py", line 532, in _locked\n return fun(self, *args, **kwargs)\n File "/odoo/odoo-server/odoo/sql_db.py", line 600, in borrow\n **connection_info)\n File "/usr/local/lib/python3.7/dist-packages/psycopg2/__init__.py", line 130, in connect\n conn = _connect(dsn, connection_factory=connection_factory, **kwasync)\npsycopg2.OperationalError: FATAL: database "pnv3" does not exist\n\n'>
Process finished with exit cod
1
Databse does exist, have triple checked my password, not sure what else to do at this point.
The url to Odoo server should include protocol part "http://" in the beginning. Strange that you get the version info at all. What do you get as output for the version?
Also you pass the url as the last parameter to authenticate method and this is not required. This should still not give the error you received.
Try your code with these two fixes and report if this helps:
import xmlrpc.client
url ="http://localhost:8069"
db = "pnv3"
username = "test"
password = "test"
common = xmlrpc.client.ServerProxy('{}/xmlrpc/2/common'.format(url))
print(common.version())
uid = common.authenticate(db, username, password, {})
print(uid)
Identical code works on my machine...
Facing the following error in a Pyramid Application while using SQLAlchemy and Zope Transaction Manager.
This is how I am creating a scoped session:
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.schema import MetaData
from sqlalchemy.orm import (
scoped_session,
sessionmaker,
)
from zope.sqlalchemy import ZopeTransactionExtension
metadata = MetaData(naming_convention=NAMING_CONVENTION)
Base = declarative_base(metadata=metadata)
DBSession = scoped_session(sessionmaker(extension=ZopeTransactionExtension(keep_session=True)))
I am syncing my session creation and removal with my requests in the following way:
#view_config(route_name='social_get_individual_assets', renderer='json', effective_principals=2, permission='edit')
def social_get_individual_assets(requestJson):
session = DBSession()
try:
body = requestJson.request.json_body
search_term = body['text']
horizontal = body['horizontal']
vertical = body['vertical']
log.info("social get individual assets request: %s", str(search_term).encode(encoding='utf_8'))
json_data = get_individual_assets(session, search_term, horizontal, vertical)
log.info("social get assets response: %s", str(search_term).encode(encoding='utf_8'))
transaction.commit()
DBSession.remove()
return json_data
except Exception as e:
session.rollback()
log.exception(e)
raise e
For some reason, I keep running into this error all the time:
2017-12-06 13:32:07,965 ERROR [invideoapp.views.default:465] An operation previously failed, with traceback:
File "/usr/lib64/python3.5/threading.py", line 882, in _bootstrap
self._bootstrap_inner()
File "/usr/lib64/python3.5/threading.py", line 914, in _bootstrap_inner
self.run()
File "/usr/lib64/python3.5/threading.py", line 862, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.5/site-packages/waitress/task.py", line 78, in handler_thread
task.service()
File "/usr/local/lib/python3.5/site-packages/waitress/channel.py", line 338, in service
task.service()
File "/usr/local/lib/python3.5/site-packages/waitress/task.py", line 169, in service
self.execute()
File "/usr/local/lib/python3.5/site-packages/waitress/task.py", line 399, in execute
app_iter = self.channel.server.application(env, start_response)
File "/usr/local/lib/python3.5/site-packages/pyramid/router.py", line 270, in __call__
response = self.execution_policy(environ, self)
File "/usr/local/lib/python3.5/site-packages/pyramid_retry/__init__.py", line 114, in retry_policy
response = router.invoke_request(request)
File "/usr/local/lib/python3.5/site-packages/pyramid/router.py", line 249, in invoke_request
response = handle_request(request)
File "/usr/local/lib/python3.5/site-packages/pyramid_tm/__init__.py", line 136, in tm_tween
response = handler(request)
File "/usr/local/lib/python3.5/site-packages/pyramid/tweens.py", line 39, in excview_tween
response = handler(request)
File "/usr/local/lib/python3.5/site-packages/pyramid/router.py", line 156, in handle_request
view_name
File "/usr/local/lib/python3.5/site-packages/pyramid/view.py", line 642, in _call_view
response = view_callable(context, request)
File "/usr/local/lib/python3.5/site-packages/pyramid/viewderivers.py", line 439, in rendered_view
result = view(context, request)
File "/usr/local/lib/python3.5/site-packages/pyramid/viewderivers.py", line 148, in _requestonly_view
response = view(request)
File "/home/ttv/invideoapp/invideoapp/views/default.py", line 148, in upload_image
transaction.commit()
File "/usr/local/lib/python3.5/site-packages/transaction/_manager.py", line 131, in commit
return self.get().commit()
File "/usr/local/lib/python3.5/site-packages/transaction/_transaction.py", line 308, in commit
t, v, tb = self._saveAndGetCommitishError()
File "/usr/local/lib/python3.5/site-packages/transaction/_transaction.py", line 301, in commit
self._commitResources()
File "/usr/local/lib/python3.5/site-packages/transaction/_transaction.py", line 446, in _commitResources
reraise(t, v, tb)
File "/usr/local/lib/python3.5/site-packages/transaction/_compat.py", line 54, in reraise
raise value
File "/usr/local/lib/python3.5/site-packages/transaction/_transaction.py", line 423, in _commitResources
rm.tpc_vote(self)
File "/usr/local/lib/python3.5/site-packages/zope/sqlalchemy/datamanager.py", line 109, in tpc_vote
self.tx.commit()
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/session.py", line 459, in commit
self._assert_active(prepared_ok=True)
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/session.py", line 285, in _assert_active
raise sa_exc.ResourceClosedError(closed_msg)
sqlalchemy.exc.ResourceClosedError: This transaction is closed
Traceback (most recent call last):
File "/home/ttv/invideoapp/invideoapp/views/default.py", line 451, in update_master_json
user = getUser(session,user_id)
File "/home/ttv/invideoapp/invideoapp/invideomodules/auth_processing.py", line 12, in getUser
raise e
File "/home/ttv/invideoapp/invideoapp/invideomodules/auth_processing.py", line 8, in getUser
query = session.query(User).filter(User.user_id == userid).first()
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/query.py", line 2755, in first
ret = list(self[0:1])
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/query.py", line 2547, in __getitem__
return list(res)
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/query.py", line 2855, in __iter__
return self._execute_and_instances(context)
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/query.py", line 2876, in _execute_and_instances
close_with_result=True)
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/query.py", line 2885, in _get_bind_args
**kw
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/query.py", line 2867, in _connection_from_session
conn = self.session.connection(**kw)
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/session.py", line 966, in connection
execution_options=execution_options)
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/session.py", line 971, in _connection_for_bind
engine, execution_options)
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/session.py", line 417, in _connection_for_bind
self.session.dispatch.after_begin(self.session, self, conn)
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/event/attr.py", line 256, in __call__
fn(*args, **kw)
File "/usr/local/lib/python3.5/site-packages/zope/sqlalchemy/datamanager.py", line 237, in after_begin
join_transaction(session, self.initial_state, self.transaction_manager, self.keep_session)
File "/usr/local/lib/python3.5/site-packages/zope/sqlalchemy/datamanager.py", line 211, in join_transaction
DataManager(session, initial_state, transaction_manager, keep_session=keep_session)
File "/usr/local/lib/python3.5/site-packages/zope/sqlalchemy/datamanager.py", line 73, in __init__
transaction_manager.get().join(self)
File "/usr/local/lib/python3.5/site-packages/transaction/_transaction.py", line 179, in join
self._prior_operation_failed() # doesn't return
File "/usr/local/lib/python3.5/site-packages/transaction/_transaction.py", line 173, in _prior_operation_failed
self._failure_traceback.getvalue())
transaction.interfaces.TransactionFailedError: An operation previously failed, with traceback:
File "/usr/lib64/python3.5/threading.py", line 882, in _bootstrap
self._bootstrap_inner()
File "/usr/lib64/python3.5/threading.py", line 914, in _bootstrap_inner
self.run()
File "/usr/lib64/python3.5/threading.py", line 862, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.5/site-packages/waitress/task.py", line 78, in handler_thread
task.service()
File "/usr/local/lib/python3.5/site-packages/waitress/channel.py", line 338, in service
task.service()
File "/usr/local/lib/python3.5/site-packages/waitress/task.py", line 169, in service
self.execute()
File "/usr/local/lib/python3.5/site-packages/waitress/task.py", line 399, in execute
app_iter = self.channel.server.application(env, start_response)
File "/usr/local/lib/python3.5/site-packages/pyramid/router.py", line 270, in __call__
response = self.execution_policy(environ, self)
File "/usr/local/lib/python3.5/site-packages/pyramid_retry/__init__.py", line 114, in retry_policy
response = router.invoke_request(request)
File "/usr/local/lib/python3.5/site-packages/pyramid/router.py", line 249, in invoke_request
response = handle_request(request)
File "/usr/local/lib/python3.5/site-packages/pyramid_tm/__init__.py", line 136, in tm_tween
response = handler(request)
File "/usr/local/lib/python3.5/site-packages/pyramid/tweens.py", line 39, in excview_tween
response = handler(request)
File "/usr/local/lib/python3.5/site-packages/pyramid/router.py", line 156, in handle_request
view_name
File "/usr/local/lib/python3.5/site-packages/pyramid/view.py", line 642, in _call_view
response = view_callable(context, request)
File "/usr/local/lib/python3.5/site-packages/pyramid/viewderivers.py", line 439, in rendered_view
result = view(context, request)
File "/usr/local/lib/python3.5/site-packages/pyramid/viewderivers.py", line 148, in _requestonly_view
response = view(request)
File "/home/ttv/invideoapp/invideoapp/views/default.py", line 148, in upload_image
transaction.commit()
File "/usr/local/lib/python3.5/site-packages/transaction/_manager.py", line 131, in commit
return self.get().commit()
File "/usr/local/lib/python3.5/site-packages/transaction/_transaction.py", line 308, in commit
t, v, tb = self._saveAndGetCommitishError()
File "/usr/local/lib/python3.5/site-packages/transaction/_transaction.py", line 301, in commit
self._commitResources()
File "/usr/local/lib/python3.5/site-packages/transaction/_transaction.py", line 446, in _commitResources
reraise(t, v, tb)
File "/usr/local/lib/python3.5/site-packages/transaction/_compat.py", line 54, in reraise
raise value
File "/usr/local/lib/python3.5/site-packages/transaction/_transaction.py", line 423, in _commitResources
rm.tpc_vote(self)
File "/usr/local/lib/python3.5/site-packages/zope/sqlalchemy/datamanager.py", line 109, in tpc_vote
self.tx.commit()
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/session.py", line 459, in commit
self._assert_active(prepared_ok=True)
File "/usr/local/lib64/python3.5/site-packages/sqlalchemy/orm/session.py", line 285, in _assert_active
raise sa_exc.ResourceClosedError(closed_msg)
sqlalchemy.exc.ResourceClosedError: This transaction is closed
Once I get this error, none of my Database connection stuff works. Only way out is to restart the entire application to make things work. Any idea why this is happening?
I had a similar problem, and I found using DBSession.begin_nested() solved the issue. So something like:
#view_config(route_name='social_get_individual_assets', renderer='json', effective_principals=2, permission='edit')
def social_get_individual_assets(requestJson):
try:
DBSession.begin_nested()
body = requestJson.request.json_body
search_term = body['text']
horizontal = body['horizontal']
vertical = body['vertical']
log.info("social get individual assets request: %s", str(search_term).encode(encoding='utf_8'))
json_data = get_individual_assets(session, search_term, horizontal, vertical)
log.info("social get assets response: %s", str(search_term).encode(encoding='utf_8'))
return json_data
except Exception as e:
DBSession.rollback()
log.exception(e)
raise e