I'm running trading bots written in python in VS Code on mac, trading at binance exchange.
Unfortunately I get sporadic errors when the bot is placing orders, respectively connect via websocket. I really can't explain what's going on because the bots are running well for one or two days and then suddenly this error is disrupting the strategy, because it causes that orders are not cancelled anymore when they should leading to unsecured trades (without stop).
Message: 'Starting new HTTPS connection (%d): %s:%s'
Arguments: (1, 'fapi.binance.com', 443)
--- Logging error ---
Traceback (most recent call last):
File "/Users/username/anaconda3/lib/python3.7/logging/__init__.py", line 1038, in emit
self.flush()
File "/Users/username/anaconda3/lib/python3.7/logging/__init__.py", line 1018, in flush
self.stream.flush()
PermissionError: [Errno 13] Permission denied
Call stack:
File "/Users/username/anaconda3/lib/python3.7/threading.py", line 885, in _bootstrap
self._bootstrap_inner()
File "/Users/username/anaconda3/lib/python3.7/threading.py", line 917, in _bootstrap_inner
self.run()
File "/Users/username/anaconda3/lib/python3.7/threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "/Users/username/Workfiles/python/binance/vs_binance_algo/ETH_LinReg/engine_LinReg.py", line 102, in execute
execution_short_ETH.main()
File "/Users/username/Workfiles/python/binance/vs_binance_algo/ETH_LinReg/execution_short_ETH.py", line 22, in main
response_tradelist = sr.send_signed_request('GET', '/fapi/v1/userTrades', params)
File "/Users/username/Workfiles/python/binance/vs_binance_algo/ETH_LinReg/send_requests_ETH.py", line 49, in send_signed_request
response = dispatch_request(http_method)(**params)
File "/Users/username/anaconda3/lib/python3.7/site-packages/requests/sessions.py", line 546, in get
return self.request('GET', url, **kwargs)
File "/Users/username/anaconda3/lib/python3.7/site-packages/requests/sessions.py", line 533, in request
resp = self.send(prep, **send_kwargs)
File "/Users/username/anaconda3/lib/python3.7/site-packages/requests/sessions.py", line 646, in send
r = adapter.send(request, **kwargs)
File "/Users/username/anaconda3/lib/python3.7/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/Users/username/anaconda3/lib/python3.7/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/Users/username/anaconda3/lib/python3.7/site-packages/urllib3/connectionpool.py", line 393, in _make_request
httplib_response.length)
Message: '%s://%s:%s "%s %s %s" %s %s'
Arguments: ('https', 'fapi.binance.com', 443, 'GET', '/fapi/v1/userTrades?timestamp=1630094893528&signature=456f8b3cf34a36ad257114d4e8a7ca31d43c27ddb8ad8296bc7e3d3d416f9145', 'HTTP/1.1', 200, None)
--- Logging error ---
Traceback (most recent call last):
File "/Users/username/anaconda3/lib/python3.7/logging/__init__.py", line 1038, in emit
self.flush()
File "/Users/username/anaconda3/lib/python3.7/logging/__init__.py", line 1018, in flush
self.stream.flush()
PermissionError: [Errno 13] Permission denied
Call stack:
File "/Users/username/anaconda3/lib/python3.7/threading.py", line 885, in _bootstrap
self._bootstrap_inner()
File "/Users/username/anaconda3/lib/python3.7/threading.py", line 917, in _bootstrap_inner
self.run()
File "/Users/username/anaconda3/lib/python3.7/threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "/Users/username/Workfiles/python/binance/vs_binance_algo/ETH_LinReg/engine_LinReg.py", line 102, in execute
execution_short_ETH.main()
File "/Users/username/Workfiles/python/binance/vs_binance_algo/ETH_LinReg/execution_short_ETH.py", line 34, in main
#response = sr.send_public_request('/fapi/v1/ticker/24hr?symbol='+symbol)
File "/Users/username/Workfiles/python/binance/vs_binance_algo/ETH_LinReg/send_requests_ETH.py", line 59, in send_public_request
response = dispatch_request('GET')(url=url)
File "/Users/username/anaconda3/lib/python3.7/site-packages/requests/sessions.py", line 546, in get
return self.request('GET', url, **kwargs)
File "/Users/username/anaconda3/lib/python3.7/site-packages/requests/sessions.py", line 533, in request
resp = self.send(prep, **send_kwargs)
File "/Users/username/anaconda3/lib/python3.7/site-packages/requests/sessions.py", line 646, in send
r = adapter.send(request, **kwargs)
File "/Users/username/anaconda3/lib/python3.7/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/Users/username/anaconda3/lib/python3.7/site-packages/urllib3/connectionpool.py", line 588, in urlopen
conn = self._get_conn(timeout=pool_timeout)
File "/Users/username/anaconda3/lib/python3.7/site-packages/urllib3/connectionpool.py", line 248, in _get_conn
return conn or self._new_conn()
File "/Users/username/anaconda3/lib/python3.7/site-packages/urllib3/connectionpool.py", line 813, in _new_conn
self.num_connections, self.host, self.port or "443")
Message: 'Starting new HTTPS connection (%d): %s:%s'
Arguments: (1, 'fapi.binance.com', 443)
--- Logging error ---
Traceback (most recent call last):
File "/Users/username/anaconda3/lib/python3.7/logging/__init__.py", line 1038, in emit
self.flush()
File "/Users/username/anaconda3/lib/python3.7/logging/__init__.py", line 1018, in flush
self.stream.flush()
PermissionError: [Errno 13] Permission denied
Call stack:
File "/Users/username/anaconda3/lib/python3.7/threading.py", line 885, in _bootstrap
self._bootstrap_inner()
File "/Users/username/anaconda3/lib/python3.7/threading.py", line 917, in _bootstrap_inner
self.run()
File "/Users/username/anaconda3/lib/python3.7/threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "/Users/username/Workfiles/python/binance/vs_binance_algo/ETH_LinReg/engine_LinReg.py", line 102, in execute
execution_short_ETH.main()
File "/Users/username/Workfiles/python/binance/vs_binance_algo/ETH_LinReg/execution_short_ETH.py", line 34, in main
#response = sr.send_public_request('/fapi/v1/ticker/24hr?symbol='+symbol)
File "/Users/username/Workfiles/python/binance/vs_binance_algo/ETH_LinReg/send_requests_ETH.py", line 59, in send_public_request
response = dispatch_request('GET')(url=url)
File "/Users/username/anaconda3/lib/python3.7/site-packages/requests/sessions.py", line 546, in get
return self.request('GET', url, **kwargs)
File "/Users/username/anaconda3/lib/python3.7/site-packages/requests/sessions.py", line 533, in request
resp = self.send(prep, **send_kwargs)
File "/Users/username/anaconda3/lib/python3.7/site-packages/requests/sessions.py", line 646, in send
r = adapter.send(request, **kwargs)
File "/Users/username/anaconda3/lib/python3.7/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/Users/username/anaconda3/lib/python3.7/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/Users/username/anaconda3/lib/python3.7/site-packages/urllib3/connectionpool.py", line 393, in _make_request
httplib_response.length)
Message: '%s://%s:%s "%s %s %s" %s %s'
Arguments: ('https', 'fapi.binance.com', 443, 'POST', '/fapi/v1/order?symbol=ETHUSDT&side=SELL&type=MARKET&quantity=0.01&newOrderRespType=RESULT×tamp=1630094894451&signature=67252a93834cedde133ad0fa62683126a9ca583147fffcf2cefbdff071fa4ce3', 'HTTP/1.1', 200, None)
--- Logging error ---
Traceback (most recent call last):
File "/Users/username/anaconda3/lib/python3.7/logging/__init__.py", line 1038, in emit
self.flush()
File "/Users/username/anaconda3/lib/python3.7/logging/__init__.py", line 1018, in flush
self.stream.flush()
PermissionError: [Errno 13] Permission denied
Call stack:
File "/Users/username/anaconda3/lib/python3.7/threading.py", line 885, in _bootstrap
self._bootstrap_inner()
File "/Users/username/anaconda3/lib/python3.7/threading.py", line 917, in _bootstrap_inner
self.run()
File "/Users/username/anaconda3/lib/python3.7/threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "/Users/username/Workfiles/python/binance/vs_binance_algo/ETH_LinReg/engine_LinReg.py", line 102, in execute
execution_short_ETH.main()
File "/Users/username/Workfiles/python/binance/vs_binance_algo/ETH_LinReg/execution_short_ETH.py", line 58, in main
TP = int(entryPrice-settings_ETH.TP)
File "/Users/username/Workfiles/python/binance/vs_binance_algo/ETH_LinReg/send_requests_ETH.py", line 49, in send_signed_request
response = dispatch_request(http_method)(**params)
File "/Users/username/anaconda3/lib/python3.7/site-packages/requests/sessions.py", line 546, in get
return self.request('GET', url, **kwargs)
File "/Users/username/anaconda3/lib/python3.7/site-packages/requests/sessions.py", line 533, in request
resp = self.send(prep, **send_kwargs)
File "/Users/username/anaconda3/lib/python3.7/site-packages/requests/sessions.py", line 646, in send
r = adapter.send(request, **kwargs)
File "/Users/username/anaconda3/lib/python3.7/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/Users/username/anaconda3/lib/python3.7/site-packages/urllib3/connectionpool.py", line 588, in urlopen
conn = self._get_conn(timeout=pool_timeout)
File "/Users/username/anaconda3/lib/python3.7/site-packages/urllib3/connectionpool.py", line 248, in _get_conn
return conn or self._new_conn()
File "/Users/username/anaconda3/lib/python3.7/site-packages/urllib3/connectionpool.py", line 813, in _new_conn
self.num_connections, self.host, self.port or "443")
...and so on...
I can't find anything in the dev.binance forum in this regard (and they are anyway not very helpful).
What does this error mean?
My intuition is an imported module try to create a log file with logging module and raise a PermissionError.
Try to disable logging:
import logging.config
logging.config.dictConfig({
'version': 1,
'disable_existing_loggers': True,
})
Related
I am trying to troubleshoot a situation.
I am initiating a Boto3 client like this:
s3_client = boto3.client('s3')
Then I am iterating over a number of files and uploading them using:
s3_client.upload_file()
My problem is that every now and then I see it pausing for 60 seconds and then continuing normally.
I've tried reducing the timeout value and the error I seem to be getting is:
botocore.exceptions.ConnectTimeoutError
or
urllib3.exceptions.ConnectTimeoutError
If I leave it at 60, eventually it succeeds and I don't get any errors.
My question is, this error means that when executing upload_file, it tried to send a put request to the server and the server didn't respond to that request?
When initiating the client, is any connection established that could be lost or that's simply only storing the credentials and it's irrelevant to this issue?
Many thanks.
Update: Adding more detailed logs:
CRITICAL Connect timeout
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 169, in _new_conn
conn = connection.create_connection(
File "/usr/lib/python3/dist-packages/urllib3/util/connection.py", line 96, in create_connection
raise err
File "/usr/lib/python3/dist-packages/urllib3/util/connection.py", line 86, in create_connection
sock.connect(sa)
TimeoutError: timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/user/.local/lib/python3.10/site-packages/botocore/httpsession.py", line 439, in send
urllib_response = conn.urlopen(
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 755, in urlopen
retries = retries.increment(
File "/usr/lib/python3/dist-packages/urllib3/util/retry.py", line 507, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/usr/lib/python3/dist-packages/six.py", line 719, in reraise
raise value
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 382, in _make_request
self._validate_conn(conn)
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 1012, in _validate_conn
conn.connect()
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 353, in connect
conn = self._new_conn()
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 174, in _new_conn
raise ConnectTimeoutError(
urllib3.exceptions.ConnectTimeoutError
During handling of the above exception, another exception occurred:
File "/home/user/.local/lib/python3.10/site-packages/boto3/s3/inject.py", line 143, in upload_file
return transfer.upload_file(
File "/home/user/.local/lib/python3.10/site-packages/boto3/s3/transfer.py", line 288, in upload_file
future.result()
File "/home/user/.local/lib/python3.10/site-packages/s3transfer/futures.py", line 103, in result
return self._coordinator.result()
File "/home/user/.local/lib/python3.10/site-packages/s3transfer/futures.py", line 266, in result
raise self._exception
File "/home/user/.local/lib/python3.10/site-packages/s3transfer/tasks.py", line 139, in __call__
return self._execute_main(kwargs)
File "/home/user/.local/lib/python3.10/site-packages/s3transfer/tasks.py", line 162, in _execute_main
return_value = self._main(**kwargs)
File "/home/user/.local/lib/python3.10/site-packages/s3transfer/upload.py", line 758, in _main
client.put_object(Bucket=bucket, Key=key, Body=body, **extra_args)
File "/home/user/.local/lib/python3.10/site-packages/botocore/client.py", line 415, in _api_call
return self._make_api_call(operation_name, kwargs)
File "/home/user/.local/lib/python3.10/site-packages/botocore/client.py", line 731, in _make_api_call
http, parsed_response = self._make_request(
File "/home/user/.local/lib/python3.10/site-packages/botocore/client.py", line 751, in _make_request
return self._endpoint.make_request(operation_model, request_dict)
File "/home/user/.local/lib/python3.10/site-packages/botocore/endpoint.py", line 107, in make_request
return self._send_request(request_dict, operation_model)
File "/home/user/.local/lib/python3.10/site-packages/botocore/endpoint.py", line 183, in _send_request
while self._needs_retry(attempts, operation_model, request_dict,
File "/home/user/.local/lib/python3.10/site-packages/botocore/endpoint.py", line 305, in _needs_retry
responses = self._event_emitter.emit(
File "/home/user/.local/lib/python3.10/site-packages/botocore/hooks.py", line 358, in emit
return self._emitter.emit(aliased_event_name, **kwargs)
File "/home/user/.local/lib/python3.10/site-packages/botocore/hooks.py", line 229, in emit
return self._emit(event_name, kwargs)
File "/home/user/.local/lib/python3.10/site-packages/botocore/hooks.py", line 212, in _emit
response = handler(**kwargs)
File "/home/user/.local/lib/python3.10/site-packages/botocore/retryhandler.py", line 194, in __call__
if self._checker(**checker_kwargs):
File "/home/user/.local/lib/python3.10/site-packages/botocore/retryhandler.py", line 267, in __call__
should_retry = self._should_retry(attempt_number, response,
File "/home/user/.local/lib/python3.10/site-packages/botocore/retryhandler.py", line 294, in _should_retry
return self._checker(attempt_number, response, caught_exception)
File "/home/user/.local/lib/python3.10/site-packages/botocore/retryhandler.py", line 333, in __call__
checker_response = checker(attempt_number, response,
File "/home/user/.local/lib/python3.10/site-packages/botocore/retryhandler.py", line 233, in __call__
return self._check_caught_exception(
File "/home/user/.local/lib/python3.10/site-packages/botocore/retryhandler.py", line 376, in _check_caught_exception
raise caught_exception
File "/home/user/.local/lib/python3.10/site-packages/botocore/endpoint.py", line 249, in _do_get_response
http_response = self._send(request)
File "/home/user/.local/lib/python3.10/site-packages/botocore/endpoint.py", line 321, in _send
return self.http_session.send(request)
File "/home/user/.local/lib/python3.10/site-packages/botocore/httpsession.py", line 472, in send
raise ConnectTimeoutError(endpoint_url=request.url, error=e)
The default boto3 retry mode (Legacy) doesn't properly handle a number of errors/exceptions, including RequestThrottled, PriorRequestNotComplete, ConnectionError and BandwidthLimitExceeded. It can manifest as very intermittent problems with long delays and repeated connection failures. You'll likely be better off using the Standard retry mode instead, which also has an exponential backoff.
import boto3
from botocore.client import Config as BotoConfig
TIMEOUT = 3
config = BotoConfig(connect_timeout=TIMEOUT, retries={"mode": "standard"})
client = boto3.client("s3", config=config)
I am writing a code to take frequent backups of our database to dropbox. The following is my tasks.py that has a test code to execute a Django management command management.call_command('dbbackup', interactive=False) every 1 minute. But I am encountering the error django.db.utils.DatabaseError: DatabaseWrapper objects created in a thread can only be used in that same thread. The object with alias 'default' was created in thread id 140423029811616 and this is thread id 140422957233632..
tasks.py
from __future__ import absolute_import, unicode_literals
from myproject.celery import app
from celery.utils.log import get_task_logger
from celery.schedules import crontab
from django.core import management
#app.task
def db_backup_to_dropbox():
management.call_command('dbbackup', interactive=False)
app.conf.beat_schedule = {
'backup-every-second': {
'task': 'core.tasks.db_backup_to_dropbox',
'schedule': crontab(minute='*/1'),
'args': (),
},
}
Traceback from celery.log
[2020-07-25 16:24:00,050: INFO/MainProcess] Received task: core.tasks.db_backup_to_dropbox[a34653fe-a3c6-4259-a98d-7613c5f3446c]
[2020-07-25 16:24:00,054: INFO/MainProcess] Backing Up Database: myproject
[2020-07-25 16:24:00,213: INFO/MainProcess] Writing file to default-myserver-loc1-2020-07-25-162400.dump
[2020-07-25 16:24:00,215: INFO/MainProcess] Request to files/get_metadata
[2020-07-25 16:24:00,287: ERROR/MainProcess] RecursionError: maximum recursion depth exceeded while calling a Python object
File "/myproject/venv/lib/python3.8/site-packages/dbbackup/utils.py", line 118, in wrapper
func(*args, **kwargs)
File "/myproject/venv/lib/python3.8/site-packages/dbbackup/management/commands/dbbackup.py", line 61, in handle
self._save_new_backup(database)
File "/myproject/venv/lib/python3.8/site-packages/dbbackup/management/commands/dbbackup.py", line 88, in _save_new_backup
self.write_to_storage(outputfile, filename)
File "/myproject/venv/lib/python3.8/site-packages/dbbackup/management/commands/_base.py", line 88, in write_to_storage
self.storage.write_file(file, path)
File "/myproject/venv/lib/python3.8/site-packages/dbbackup/storage.py", line 82, in write_file
self.storage.save(name=filename, content=filehandle)
File "/myproject/venv/lib/python3.8/site-packages/django/core/files/storage.py", line 51, in save
name = self.get_available_name(name, max_length=max_length)
File "/myproject/venv/lib/python3.8/site-packages/django/core/files/storage.py", line 82, in get_available_name
while self.exists(name) or (max_length and len(name) > max_length):
File "/myproject/venv/lib/python3.8/site-packages/storages/backends/dropbox.py", line 93, in exists
return bool(self.client.files_get_metadata(self._full_path(name)))
File "/myproject/venv/lib/python3.8/site-packages/dropbox/base.py", line 1682, in files_get_metadata
r = self.request(
File "/myproject/venv/lib/python3.8/site-packages/dropbox/dropbox.py", line 313, in request
res = self.request_json_string_with_retry(host,
File "/myproject/venv/lib/python3.8/site-packages/dropbox/dropbox.py", line 463, in request_json_string_with_retry
return self.request_json_string(host,
File "/myproject/venv/lib/python3.8/site-packages/dropbox/dropbox.py", line 559, in request_json_string
r = self._session.post(url,
File "/myproject/venv/lib/python3.8/site-packages/requests/sessions.py", line 578, in post
return self.request('POST', url, data=data, json=json, **kwargs)
File "/myproject/venv/lib/python3.8/site-packages/requests/sessions.py", line 530, in request
resp = self.send(prep, **send_kwargs)
File "/myproject/venv/lib/python3.8/site-packages/requests/sessions.py", line 643, in send
r = adapter.send(request, **kwargs)
File "/myproject/venv/lib/python3.8/site-packages/requests/adapters.py", line 439, in send
resp = conn.urlopen(
File "/myproject/venv/lib/python3.8/site-packages/urllib3/connectionpool.py", line 670, in urlopen httplib_response = self._make_request(
File "/myproject/venv/lib/python3.8/site-packages/urllib3/connectionpool.py", line 381, in _make_request
self._validate_conn(conn)
File "/myproject/venv/lib/python3.8/site-packages/urllib3/connectionpool.py", line 978, in _validate_conn
conn.connect()
File "/myproject/venv/lib/python3.8/site-packages/urllib3/connection.py", line 343, in connect
self.ssl_context = create_urllib3_context(
File "/myproject/venv/lib/python3.8/site-packages/urllib3/util/ssl_.py", line 277, in create_urllib3_context
context.options |= options
File "/usr/lib/python3.8/ssl.py", line 602, in options
super(SSLContext, SSLContext).options.__set__(self, value)
File "/usr/lib/python3.8/ssl.py", line 602, in options
super(SSLContext, SSLContext).options.__set__(self, value)
File "/usr/lib/python3.8/ssl.py", line 602, in options
super(SSLContext, SSLContext).options.__set__(self, value)
[Previous line repeated 460 more times]
[2020-07-25 16:24:00,325: ERROR/MainProcess] Task core.tasks.db_backup_to_dropbox[a34653fe-a3c6-4259-a98d-7613c5f3446c] raised unexpected: DatabaseError("DatabaseWrapper objects created in a thread can only be used in that same thread. The object with alias 'default' was created in thread id 140423029811616 and this is thread id 140422957233632.")
Traceback (most recent call last):
File "/myproject/venv/lib/python3.8/site-packages/dbbackup/utils.py", line 118, in wrapper
func(*args, **kwargs)
File "/myproject/venv/lib/python3.8/site-packages/dbbackup/management/commands/dbbackup.py", line 61, in handle
self._save_new_backup(database)
File "/myproject/venv/lib/python3.8/site-packages/dbbackup/management/commands/dbbackup.py", line 88, in _save_new_backup
self.write_to_storage(outputfile, filename)
File "/myproject/venv/lib/python3.8/site-packages/dbbackup/management/commands/_base.py", line 88, in write_to_storage
self.storage.write_file(file, path)
File "/myproject/venv/lib/python3.8/site-packages/dbbackup/storage.py", line 82, in write_file
self.storage.save(name=filename, content=filehandle)
File "/myproject/venv/lib/python3.8/site-packages/django/core/files/storage.py", line 51, in save
name = self.get_available_name(name, max_length=max_length)
File "/myproject/venv/lib/python3.8/site-packages/django/core/files/storage.py", line 82, in get_available_name
while self.exists(name) or (max_length and len(name) > max_length):
File "/myproject/venv/lib/python3.8/site-packages/storages/backends/dropbox.py", line 93, in exists
return bool(self.client.files_get_metadata(self._full_path(name)))
File "/myproject/venv/lib/python3.8/site-packages/dropbox/base.py", line 1682, in files_get_metadata
r = self.request(
File "/myproject/venv/lib/python3.8/site-packages/dropbox/dropbox.py", line 313, in request
res = self.request_json_string_with_retry(host,
File "/myproject/venv/lib/python3.8/site-packages/dropbox/dropbox.py", line 463, in request_json_string_with_retry
return self.request_json_string(host,
File "/myproject/venv/lib/python3.8/site-packages/dropbox/dropbox.py", line 559, in request_json_string
r = self._session.post(url,
File "/myproject/venv/lib/python3.8/site-packages/requests/sessions.py", line 578, in post
return self.request('POST', url, data=data, json=json, **kwargs)
File "/myproject/venv/lib/python3.8/site-packages/requests/sessions.py", line 530, in request
resp = self.send(prep, **send_kwargs)
File "/myproject/venv/lib/python3.8/site-packages/requests/sessions.py", line 643, in send
r = adapter.send(request, **kwargs)
File "/myproject/venv/lib/python3.8/site-packages/requests/adapters.py", line 439, in send
resp = conn.urlopen(
File "/myproject/venv/lib/python3.8/site-packages/urllib3/connectionpool.py", line 670, in urlopen httplib_response = self._make_request(
File "/myproject/venv/lib/python3.8/site-packages/urllib3/connectionpool.py", line 381, in _make_request
self._validate_conn(conn)
File "/myproject/venv/lib/python3.8/site-packages/urllib3/connectionpool.py", line 978, in _validate_conn
conn.connect()
File "/myproject/venv/lib/python3.8/site-packages/urllib3/connection.py", line 343, in connect
self.ssl_context = create_urllib3_context(
File "/myproject/venv/lib/python3.8/site-packages/urllib3/util/ssl_.py", line 277, in create_urllib3_context
context.options |= options
File "/usr/lib/python3.8/ssl.py", line 602, in options
super(SSLContext, SSLContext).options.__set__(self, value)
File "/usr/lib/python3.8/ssl.py", line 602, in options
super(SSLContext, SSLContext).options.__set__(self, value)
File "/usr/lib/python3.8/ssl.py", line 602, in options
super(SSLContext, SSLContext).options.__set__(self, value)
[Previous line repeated 460 more times]
RecursionError: maximum recursion depth exceeded while calling a Python object
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/myproject/venv/lib/python3.8/site-packages/celery/app/trace.py", line 412, in trace_task
R = retval = fun(*args, **kwargs)
File "/myproject/venv/lib/python3.8/site-packages/celery/app/trace.py", line 704, in __protected_call__
return self.run(*args, **kwargs)
File "/myproject/myproject/core/tasks.py", line 11, in db_backup_to_dropbox
management.call_command('dbbackup', interactive=False)
File "/myproject/venv/lib/python3.8/site-packages/django/core/management/__init__.py", line 168, in call_command
return command.execute(*args, **defaults)
File "/myproject/venv/lib/python3.8/site-packages/django/core/management/base.py", line 369, in execute
output = self.handle(*args, **options)
File "/myproject/venv/lib/python3.8/site-packages/dbbackup/utils.py", line 127, in wrapper
connection.close()
File "/myproject/venv/lib/python3.8/site-packages/django/utils/asyncio.py", line 26, in inner
return func(*args, **kwargs)
File "/myproject/venv/lib/python3.8/site-packages/django/db/backends/base/base.py", line 286, in close
self.validate_thread_sharing()
File "/myproject/venv/lib/python3.8/site-packages/django/db/backends/base/base.py", line 553, in validate_thread_sharing
raise DatabaseError(
django.db.utils.DatabaseError: DatabaseWrapper objects created in a thread can only be used in that same thread. The object with alias 'default' was created in thread id 140423029811616 and this is thread id 140422957233632.
Try adding these lines in setting.py
CELERY_TASK_ALWAYS_EAGER = True
We have a kubernetes cron job on GCP that submits several copies of the same Python dataflow job, each in its own container. Whenever we need a new copy of the job, we just add it to the spec->jobTemplate->spec->template->spec->containers part of the cron job yaml and adjust the dataflow job parameters as needed. This usually works fine, but the latest copy we tried to add does not work. The existing copies are all still working as expected. The job seems to fail on submission to GCP, and the error message is not very helpful:
Traceback (most recent call last):
File "/app/job.py", line 117, in <module>
newness.pipeline.run_dataflow(sys.argv)
File "/app/newness/pipeline.py", line 480, in run_dataflow
result = pipe.run()
File "/usr/local/lib/python2.7/dist-packages/apache_beam/pipeline.py", line 403, in run
self.to_runner_api(), self.runner, self._options).run(False)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/pipeline.py", line 416, in run
return self.runner.run_pipeline(self)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/dataflow/dataflow_runner.py", line 389, in run_pipeline
self.dataflow_client.create_job(self.job), self)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/utils/retry.py", line 184, in wrapper
return fun(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/dataflow/internal/apiclient.py", line 504, in create_job
return self.submit_job_description(job)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/utils/retry.py", line 184, in wrapper
return fun(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/dataflow/internal/apiclient.py", line 551, in submit_job_description
response = self._client.projects_locations_jobs.Create(request)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py", line 578, in Create
config, request, global_params=global_params)
File "/usr/local/lib/python2.7/dist-packages/apitools/base/py/base_api.py", line 731, in _RunMethod
return self.ProcessHttpResponse(method_config, http_response, request)
File "/usr/local/lib/python2.7/dist-packages/apitools/base/py/base_api.py", line 737, in ProcessHttpResponse
self.__ProcessHttpResponse(method_config, http_response, request))
File "/usr/local/lib/python2.7/dist-packages/apitools/base/py/base_api.py", line 604, in __ProcessHttpResponse
http_response, method_config=method_config, request=request)
apitools.base.py.exceptions.HttpError: <exception str() failed>
The job does not appear in the dataflow console at all.
The previous lines of the container logs look like:
2019-10-13T03:57:47.725542287Z Successfully downloaded apache-beam
2019-10-13T03:58:17.125601280Z INFO:root:Staging SDK sources from PyPI to gs://gcs-bucket-name/staging/newness-boosting-c2898.1570936519.827087/dataflow_python_sdk.tar
2019-10-13T03:58:17.324843623Z INFO:root:Starting GCS upload to gs://gcs-bucket-name/staging/newness-boosting-c2898.1570936519.827087/dataflow_python_sdk.tar...
2019-10-13T03:58:24.825657227Z INFO:root:Completed GCS upload to gs://gcs-bucket-name/staging/newness-boosting-c2898.1570936519.827087/dataflow_python_sdk.tar
2019-10-13T03:58:25.225646529Z INFO:root:Downloading binary distribtution of the SDK from PyPi
2019-10-13T03:58:25.225716554Z INFO:root:Executing command: ['/usr/bin/python', '-m', 'pip', 'download', '--dest', '/tmp/tmpk5TfMS', 'apache-beam==2.8.0', '--no-deps', '--only-binary', ':all:', '--python-version', '27', '--implementation', 'cp', '--abi', 'cp27mu', '--platform', 'manylinux1_x86_64']
2019-10-13T03:59:33.926186906Z Collecting apache-beam==2.8.0
2019-10-13T03:59:52.125678183Z Using cached https://files.pythonhosted.org/packages/0f/63/ea5453ba656d060936acf41d2ec057f23aafd69649e2129ac66fdda67d48/apache_beam-2.8.0-cp27-cp27mu-manylinux1_x86_64.whl
2019-10-13T04:00:11.525435891Z Saved /tmp/tmpk5TfMS/apache_beam-2.8.0-cp27-cp27mu-manylinux1_x86_64.whl
2019-10-13T04:00:12.025054706Z Successfully downloaded apache-beam
2019-10-13T04:00:26.726190542Z INFO:root:Staging binary distribution of the SDK from PyPI to gs://gcs-bucket-name/staging/newness-boosting-c2898.1570936519.827087/apache_beam-2.8.0-cp27-cp27mu-manylinux1_x86_64.whl
2019-10-13T04:00:26.825618945Z INFO:root:Starting GCS upload to gs://gcs-bucket-name/staging/newness-boosting-c2898.1570936519.827087/apache_beam-2.8.0-cp27-cp27mu-manylinux1_x86_64.whl...
2019-10-13T04:00:33.725522899Z INFO:root:Completed GCS upload to gs://gcs-bucket-name/staging/newness-boosting-c2898.1570936519.827087/apache_beam-2.8.0-cp27-cp27mu-manylinux1_x86_64.whl
2019-10-13T04:06:14.525017097Z Traceback (most recent call last):
...
Why is this job failing to submit? Are there any other logs we can look at to see the cause of this failure?
(Most of our dataflow jobs are written in Java, where the error messages are usually more helpful.)
UPDATE: Running job locally (Windows) with apache-beam 2.16 has the same issue but more logging detail:
...
INFO:root:Starting GCS upload to gs://gcs-bucket-name/staging/newness-boosting-c2898.1571606418.971000/apache_beam-2.16.0-cp27-cp27mu-manylinux1_x86_64.whl...
INFO:root:Completed GCS upload to gs://gcs-bucket-name/staging/newness-boosting-c2898.1571606418.971000/apache_beam-2.16.0-cp27-cp27mu-manylinux1_x86_64.whl in 3 seconds.
WARNING:root:Discarding unparseable args: ['job.py', '--days_history=30']
WARNING:root:Discarding unparseable args: ['job.py', '--days_history=30']
WARNING:root:Retry with exponential backoff: waiting for 2.64795143823 seconds before retrying submit_job_description because we caught exception: error: [Errno 10053] An established connection was aborted by the software in your host machine
Traceback for above exception (most recent call last):
File "C:\Python27\lib\site-packages\apache_beam\utils\retry.py", line 206, in wrapper
return fun(*args, **kwargs)
File "C:\Python27\lib\site-packages\apache_beam\runners\dataflow\internal\apiclient.py", line 593, in submit_job_description
response = self._client.projects_locations_jobs.Create(request)
File "C:\Python27\lib\site-packages\apache_beam\runners\dataflow\internal\clients\dataflow\dataflow_v1b3_client.py", line 657, in Create
config, request, global_params=global_params)
File "C:\Python27\lib\site-packages\apitools\base\py\base_api.py", line 729, in _RunMethod
http, http_request, **opts)
File "C:\Python27\lib\site-packages\apitools\base\py\http_wrapper.py", line 346, in MakeRequest
check_response_func=check_response_func)
File "C:\Python27\lib\site-packages\apitools\base\py\http_wrapper.py", line 396, in _MakeRequestNoRetry
redirections=redirections, connection_type=connection_type)
File "C:\Python27\lib\site-packages\oauth2client\transport.py", line 169, in new_request
redirections, connection_type)
File "C:\Python27\lib\site-packages\oauth2client\transport.py", line 169, in new_request
redirections, connection_type)
File "C:\Python27\lib\site-packages\httplib2\__init__.py", line 1694, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "C:\Python27\lib\site-packages\httplib2\__init__.py", line 1434, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "C:\Python27\lib\site-packages\httplib2\__init__.py", line 1390, in _conn_request
response = conn.getresponse()
File "C:\Python27\lib\httplib.py", line 1121, in getresponse
response.begin()
File "C:\Python27\lib\httplib.py", line 438, in begin
version, status, reason = self._read_status()
File "C:\Python27\lib\httplib.py", line 394, in _read_status
line = self.fp.readline(_MAXLINE + 1)
File "C:\Python27\lib\socket.py", line 480, in readline
data = self._sock.recv(self._rbufsize)
File "C:\Python27\lib\ssl.py", line 754, in recv
return self.read(buflen)
File "C:\Python27\lib\ssl.py", line 641, in read
v = self._sslobj.read(len)
... retries 4 times total ...
Traceback (most recent call last):
File "job.py", line 117, in <module>
newness.pipeline.run_dataflow(sys.argv)
File "C:\Users\LeeW\Desktop\newness\newness\pipeline.py", line 480, in run_dataflow
result = pipe.run()
File "C:\Python27\lib\site-packages\apache_beam\pipeline.py", line 407, in run
self._options).run(False)
File "C:\Python27\lib\site-packages\apache_beam\pipeline.py", line 420, in run
return self.runner.run_pipeline(self, self._options)
File "C:\Python27\lib\site-packages\apache_beam\runners\dataflow\dataflow_runner.py", line 485, in run_pipeline
self.dataflow_client.create_job(self.job), self)
File "C:\Python27\lib\site-packages\apache_beam\utils\retry.py", line 206, in wrapper
return fun(*args, **kwargs)
File "C:\Python27\lib\site-packages\apache_beam\runners\dataflow\internal\apiclient.py", line 546, in create_job
return self.submit_job_description(job)
File "C:\Python27\lib\site-packages\apache_beam\utils\retry.py", line 219, in wrapper
raise_with_traceback(exn, exn_traceback)
File "C:\Python27\lib\site-packages\apache_beam\utils\retry.py", line 206, in wrapper
return fun(*args, **kwargs)
File "C:\Python27\lib\site-packages\apache_beam\runners\dataflow\internal\apiclient.py", line 593, in submit_job_description
response = self._client.projects_locations_jobs.Create(request)
File "C:\Python27\lib\site-packages\apache_beam\runners\dataflow\internal\clients\dataflow\dataflow_v1b3_client.py", line 657, in Create
config, request, global_params=global_params)
File "C:\Python27\lib\site-packages\apitools\base\py\base_api.py", line 729, in _RunMethod
http, http_request, **opts)
File "C:\Python27\lib\site-packages\apitools\base\py\http_wrapper.py", line 346, in MakeRequest
check_response_func=check_response_func)
File "C:\Python27\lib\site-packages\apitools\base\py\http_wrapper.py", line 396, in _MakeRequestNoRetry
redirections=redirections, connection_type=connection_type)
File "C:\Python27\lib\site-packages\oauth2client\transport.py", line 169, in new_request
redirections, connection_type)
File "C:\Python27\lib\site-packages\oauth2client\transport.py", line 169, in new_request
redirections, connection_type)
File "C:\Python27\lib\site-packages\httplib2\__init__.py", line 1694, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "C:\Python27\lib\site-packages\httplib2\__init__.py", line 1434, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "C:\Python27\lib\site-packages\httplib2\__init__.py", line 1390, in _conn_request
response = conn.getresponse()
File "C:\Python27\lib\httplib.py", line 1121, in getresponse
response.begin()
File "C:\Python27\lib\httplib.py", line 438, in begin
version, status, reason = self._read_status()
File "C:\Python27\lib\httplib.py", line 394, in _read_status
line = self.fp.readline(_MAXLINE + 1)
File "C:\Python27\lib\socket.py", line 480, in readline
data = self._sock.recv(self._rbufsize)
File "C:\Python27\lib\ssl.py", line 754, in recv
return self.read(buflen)
File "C:\Python27\lib\ssl.py", line 641, in read
v = self._sslobj.read(len)
socket.error: [Errno 10053] An established connection was aborted by the software in your host machine
Which version of Beam Python SDK are you using?
I have two python scripts, currentdataupload.py and productioncount.py on my Ubuntu server. currentdataupload.py uploads dummy current data and productioncount.py accesses that data to do some calculations. The code runs perfectly in other scenarios without disrupting the MongoDB, but when I run both of these codes simultaneously using nohup after a few seconds it exits the mongoDB service and is no longer able to upload or retrieve data. I would like to know if I am doing the entire process the right way or I am messing up somewhere big time.
Error log
Traceback (most recent call last):
File "/python_codes/currentdataupload.py", line 38, in <module>
result = posts.insert_one(post_data)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/collection.py", line 630, in insert_one
bypass_doc_val=bypass_document_validation),
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/collection.py", line 535, in _insert
check_keys, manipulate, write_concern, op_id, bypass_doc_val)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/collection.py", line 516, in _insert_one
check_keys=check_keys)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 244, in command
self._raise_connection_failure(error)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 372, in _raise_connection_failure
raise error
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 239, in command
read_concern)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/network.py", line 96, in command
response = receive_message(sock, 1, request_id)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/network.py", line 123, in receive_message
header = _receive_data_on_socket(sock, 16)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/network.py", line 161, in _receive_data_on_socket
raise AutoReconnect("connection closed")
pymongo.errors.AutoReconnect: connection closed
Traceback (most recent call last):
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 543, in connect
sock = _configured_socket(self.address, self.opts)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 452, in _configured_socket
sock = _create_connection(address, options)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 436, in _create_connection
raise err
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 429, in _create_connection
sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/python_codes/productioncount.py", line 267, in <module>
result = col2.find_one({"date": date, "machine": machine})
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/collection.py", line 1014, in find_one
for result in cursor.limit(-1):
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/cursor.py", line 1090, in next
if len(self.__data) or self._refresh():
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/cursor.py", line 1012, in _refresh
self.__read_concern))
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/cursor.py", line 850, in __send_message
**kwargs)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/mongo_client.py", line 844, in _send_message_with_response
exhaust)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/mongo_client.py", line 855, in _reset_on_error
return func(*args, **kwargs)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/server.py", line 99, in send_message_with_response
with self.get_socket(all_credentials, exhaust) as sock_info:
File "/root/anaconda3/lib/python3.5/contextlib.py", line 59, in __enter__
return next(self.gen)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/server.py", line 163, in get_socket
with self.pool.get_socket(all_credentials, checkout) as sock_info:
File "/root/anaconda3/lib/python3.5/contextlib.py", line 59, in __enter__
return next(self.gen)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 582, in get_socket
sock_info = self._get_socket_no_auth()
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 628, in _get_socket_no_auth
sock_info = self._check(sock_info)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 682, in _check
return self.connect()
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 555, in connect
_raise_connection_failure(self.address, error)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 65, in _raise_connection_failure
raise AutoReconnect(msg)
pymongo.errors.AutoReconnect: localhost:27017: [Errno 111] Connection refused
MongoDB shouldn't crash. Please file a bug report, including the Python script you run and a complete MongoDB logfile, at jira.mongodb.org.
I initialized Riak in a VM and I want use it on Windows in a Python console.
Riak port: 8098, VM host: 192.168.78.128
Windows :
from riak import RiakClient, RiakNode
client = RiakClient(protocol='http', host='192.168.78.128', http_port=8098)
bucket = client.bucket_type('visitors').bucket('hits')
bucket.new("one", data={'value':'one'}, content_type="application/json").store()
output :
Traceback (most recent call last):
File "G:/untitled3/riak_.py", line 13, in <module>
content_type="application/json").store()
File "build\bdist.win32\egg\riak\bucket.py", line 178, in new
File "build\bdist.win32\egg\riak\util.py", line 98, in __get__
File "build\bdist.win32\egg\riak\bucket.py", line 743, in datatype
File "build\bdist.win32\egg\riak\bucket.py", line 675, in get_properties
File "build\bdist.win32\egg\riak\client\transport.py", line 223, in wrapper
File "build\bdist.win32\egg\riak\client\transport.py", line 150, in _with_retries
File "C:\Python27\lib\contextlib.py", line 17, in __enter__
return self.gen.next()
File "build\bdist.win32\egg\riak\transports\pool.py", line 181, in transaction
File "build\bdist.win32\egg\riak\transports\pool.py", line 143, in acquire
File "build\bdist.win32\egg\riak\transports\http\__init__.py", line 155, in create_resource
File "build\bdist.win32\egg\riak\transports\http\transport.py", line 52, in __init__
File "build\bdist.win32\egg\riak\transports\http\connection.py", line 72, in _connect
File "build\bdist.win32\egg\riak\util.py", line 98, in __get__
File "build\bdist.win32\egg\riak\transports\feature_detect.py", line 197, in server_version
File "build\bdist.win32\egg\riak\transports\http\transport.py", line 74, in _server_version
File "build\bdist.win32\egg\riak\transports\http\transport.py", line 65, in stats
File "build\bdist.win32\egg\riak\transports\http\resources.py", line 23, in stats_path
File "build\bdist.win32\egg\riak\util.py", line 98, in __get__
File "build\bdist.win32\egg\riak\transports\http\resources.py", line 222, in riak_kv_wm_stats
File "build\bdist.win32\egg\riak\util.py", line 98, in __get__
File "build\bdist.win32\egg\riak\transports\http\resources.py", line 258, in resources
File "build\bdist.win32\egg\riak\transports\http\transport.py", line 95, in get_resources
File "build\bdist.win32\egg\riak\transports\http\connection.py", line 33, in _request
File "C:\Python27\lib\httplib.py", line 1001, in request
self._send_request(method, url, body, headers)
File "C:\Python27\lib\httplib.py", line 1035, in _send_request
self.endheaders(body)
File "C:\Python27\lib\httplib.py", line 997, in endheaders
self._send_output(message_body)
File "C:\Python27\lib\httplib.py", line 850, in _send_output
self.send(msg)
File "C:\Python27\lib\httplib.py", line 812, in send
self.connect()
File "build\bdist.win32\egg\riak\transports\http\__init__.py", line 42, in connect
File "C:\Python27\lib\httplib.py", line 793, in connect
self.timeout, self.source_address)
File "C:\Python27\lib\socket.py", line 571, in create_connection
raise err
socket.error: [Errno 10060] A connection attempt failed because the
connected party did not properly respond after a period of time, or established
connection failed because connected host has failed to respond