Develop Reference

Boto3 timeout troubleshooting

I am trying to troubleshoot a situation.
I am initiating a Boto3 client like this:
s3_client = boto3.client('s3')
Then I am iterating over a number of files and uploading them using:
s3_client.upload_file()
My problem is that every now and then I see it pausing for 60 seconds and then continuing normally.
I've tried reducing the timeout value and the error I seem to be getting is:
botocore.exceptions.ConnectTimeoutError
or
urllib3.exceptions.ConnectTimeoutError
If I leave it at 60, eventually it succeeds and I don't get any errors.
My question is, this error means that when executing upload_file, it tried to send a put request to the server and the server didn't respond to that request?
When initiating the client, is any connection established that could be lost or that's simply only storing the credentials and it's irrelevant to this issue?
Many thanks.
Update: Adding more detailed logs:
CRITICAL Connect timeout
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 169, in _new_conn
conn = connection.create_connection(
File "/usr/lib/python3/dist-packages/urllib3/util/connection.py", line 96, in create_connection
raise err
File "/usr/lib/python3/dist-packages/urllib3/util/connection.py", line 86, in create_connection
sock.connect(sa)
TimeoutError: timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/user/.local/lib/python3.10/site-packages/botocore/httpsession.py", line 439, in send
urllib_response = conn.urlopen(
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 755, in urlopen
retries = retries.increment(
File "/usr/lib/python3/dist-packages/urllib3/util/retry.py", line 507, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/usr/lib/python3/dist-packages/six.py", line 719, in reraise
raise value
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 382, in _make_request
self._validate_conn(conn)
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 1012, in _validate_conn
conn.connect()
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 353, in connect
conn = self._new_conn()
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 174, in _new_conn
raise ConnectTimeoutError(
urllib3.exceptions.ConnectTimeoutError
During handling of the above exception, another exception occurred:
File "/home/user/.local/lib/python3.10/site-packages/boto3/s3/inject.py", line 143, in upload_file
return transfer.upload_file(
File "/home/user/.local/lib/python3.10/site-packages/boto3/s3/transfer.py", line 288, in upload_file
future.result()
File "/home/user/.local/lib/python3.10/site-packages/s3transfer/futures.py", line 103, in result
return self._coordinator.result()
File "/home/user/.local/lib/python3.10/site-packages/s3transfer/futures.py", line 266, in result
raise self._exception
File "/home/user/.local/lib/python3.10/site-packages/s3transfer/tasks.py", line 139, in __call__
return self._execute_main(kwargs)
File "/home/user/.local/lib/python3.10/site-packages/s3transfer/tasks.py", line 162, in _execute_main
return_value = self._main(**kwargs)
File "/home/user/.local/lib/python3.10/site-packages/s3transfer/upload.py", line 758, in _main
client.put_object(Bucket=bucket, Key=key, Body=body, **extra_args)
File "/home/user/.local/lib/python3.10/site-packages/botocore/client.py", line 415, in _api_call
return self._make_api_call(operation_name, kwargs)
File "/home/user/.local/lib/python3.10/site-packages/botocore/client.py", line 731, in _make_api_call
http, parsed_response = self._make_request(
File "/home/user/.local/lib/python3.10/site-packages/botocore/client.py", line 751, in _make_request
return self._endpoint.make_request(operation_model, request_dict)
File "/home/user/.local/lib/python3.10/site-packages/botocore/endpoint.py", line 107, in make_request
return self._send_request(request_dict, operation_model)
File "/home/user/.local/lib/python3.10/site-packages/botocore/endpoint.py", line 183, in _send_request
while self._needs_retry(attempts, operation_model, request_dict,
File "/home/user/.local/lib/python3.10/site-packages/botocore/endpoint.py", line 305, in _needs_retry
responses = self._event_emitter.emit(
File "/home/user/.local/lib/python3.10/site-packages/botocore/hooks.py", line 358, in emit
return self._emitter.emit(aliased_event_name, **kwargs)
File "/home/user/.local/lib/python3.10/site-packages/botocore/hooks.py", line 229, in emit
return self._emit(event_name, kwargs)
File "/home/user/.local/lib/python3.10/site-packages/botocore/hooks.py", line 212, in _emit
response = handler(**kwargs)
File "/home/user/.local/lib/python3.10/site-packages/botocore/retryhandler.py", line 194, in __call__
if self._checker(**checker_kwargs):
File "/home/user/.local/lib/python3.10/site-packages/botocore/retryhandler.py", line 267, in __call__
should_retry = self._should_retry(attempt_number, response,
File "/home/user/.local/lib/python3.10/site-packages/botocore/retryhandler.py", line 294, in _should_retry
return self._checker(attempt_number, response, caught_exception)
File "/home/user/.local/lib/python3.10/site-packages/botocore/retryhandler.py", line 333, in __call__
checker_response = checker(attempt_number, response,
File "/home/user/.local/lib/python3.10/site-packages/botocore/retryhandler.py", line 233, in __call__
return self._check_caught_exception(
File "/home/user/.local/lib/python3.10/site-packages/botocore/retryhandler.py", line 376, in _check_caught_exception
raise caught_exception
File "/home/user/.local/lib/python3.10/site-packages/botocore/endpoint.py", line 249, in _do_get_response
http_response = self._send(request)
File "/home/user/.local/lib/python3.10/site-packages/botocore/endpoint.py", line 321, in _send
return self.http_session.send(request)
File "/home/user/.local/lib/python3.10/site-packages/botocore/httpsession.py", line 472, in send
raise ConnectTimeoutError(endpoint_url=request.url, error=e)

The default boto3 retry mode (Legacy) doesn't properly handle a number of errors/exceptions, including RequestThrottled, PriorRequestNotComplete, ConnectionError and BandwidthLimitExceeded. It can manifest as very intermittent problems with long delays and repeated connection failures. You'll likely be better off using the Standard retry mode instead, which also has an exponential backoff.
import boto3
from botocore.client import Config as BotoConfig
TIMEOUT = 3
config = BotoConfig(connect_timeout=TIMEOUT, retries={"mode": "standard"})
client = boto3.client("s3", config=config)

Is there a way to solve an ssl error (DH key too small) on python when using pd.read_excel()?

I am trying to read an excel table that can be accessed through an url, and load it into a DataFrame.
I'm using pd.read_excel like so:
url = "https://www.recope.go.cr/wp-content/uploads/2020/09/PRECIOS-HIST%C3%93RICOS-CONSUMIDOR-FINAL-1-2.xls"
df = pd.read_excel(url,header=[2,3,4],nrows=347)
and I get :
Traceback (most recent call last):
File "/usr/local/lib/python3.7/urllib/request.py", line 1350, in do_open
encode_chunked=req.has_header('Transfer-encoding'))
File "/usr/local/lib/python3.7/http/client.py", line 1277, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/local/lib/python3.7/http/client.py", line 1323, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/local/lib/python3.7/http/client.py", line 1272, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/local/lib/python3.7/http/client.py", line 1032, in _send_output
self.send(msg)
File "/usr/local/lib/python3.7/http/client.py", line 972, in send
self.connect()
File "/usr/local/lib/python3.7/http/client.py", line 1447, in connect
server_hostname=server_hostname)
File "/usr/local/lib/python3.7/ssl.py", line 423, in wrap_socket
session=session
File "/usr/local/lib/python3.7/ssl.py", line 870, in _create
self.do_handshake()
File "/usr/local/lib/python3.7/ssl.py", line 1139, in do_handshake
self._sslobj.do_handshake()
ssl.SSLError: [SSL: DH_KEY_TOO_SMALL] dh key too small (_ssl.c:1091)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/local/lib/python3.7/site-packages/pandas/util/_decorators.py", line 299, in wrapper
return func(*args, **kwargs)
File "/usr/local/lib/python3.7/site-packages/pandas/io/excel/_base.py", line 336, in read_excel
io = ExcelFile(io, storage_options=storage_options, engine=engine)
File "/usr/local/lib/python3.7/site-packages/pandas/io/excel/_base.py", line 1063, in __init__
path=str(self._io), storage_options=storage_options
File "/usr/local/lib/python3.7/site-packages/pandas/io/excel/_base.py", line 939, in inspect_excel_format
content_or_path, "rb", storage_options=storage_options, is_text=False
File "/usr/local/lib/python3.7/site-packages/pandas/io/common.py", line 563, in get_handle
storage_options=storage_options,
File "/usr/local/lib/python3.7/site-packages/pandas/io/common.py", line 288, in _get_filepath_or_buffer
req = urlopen(filepath_or_buffer)
File "/usr/local/lib/python3.7/site-packages/pandas/io/common.py", line 194, in urlopen
return urllib.request.urlopen(*args, **kwargs)
File "/usr/local/lib/python3.7/urllib/request.py", line 222, in urlopen
return opener.open(url, data, timeout)
File "/usr/local/lib/python3.7/urllib/request.py", line 525, in open
response = self._open(req, data)
File "/usr/local/lib/python3.7/urllib/request.py", line 543, in _open
'_open', req)
File "/usr/local/lib/python3.7/urllib/request.py", line 503, in _call_chain
result = func(*args)
File "/usr/local/lib/python3.7/urllib/request.py", line 1393, in https_open
context=self._context, check_hostname=self._check_hostname)
File "/usr/local/lib/python3.7/urllib/request.py", line 1352, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error [SSL: DH_KEY_TOO_SMALL] dh key too small (_ssl.c:1091)>
I saw something similar in this question but I tried to include the answer code in mine to no avail. I'm a bit new to this but as I couldn't find a solution I thought best to ask here. Is there any way to get around this ssl error?

I tried the solution from the link again and it worked, but I had to change some of the code to:
req = requests.get(url)
df = pd.read_excel(req.content,header=[2,3,4],nrows=347)
For some reason, adding requests made the solution work.

Issues with Firebase Admin SDK on elastic beanstalk

Just to give a brief introduction, I am using elastic beanstalk from Amazon to deploy my python (Falcon) application. I have two environments of elastic beanstalk (one being the main app server and the other one a worker environment). The worker environment receives HTTP requests for async tasks and then executes them. Most of the tasks are either push notifications (mobile devices) or sending emails. The strange part is that my tasks are failing with errors that I am unable to determine the cause for (this works fine for my local environment). Some of the tracebacks look like this:
self.send_message(message=message)
File "/var/app/current/application/services/push.py", line 114, in send_message
response = messaging.send_multicast(message)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/firebase_admin/messaging.py", line 167, in send_multicast
return _get_messaging_service(app).send_all(messages, dry_run)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/firebase_admin/messaging.py", line 390, in send_all
batch.execute()
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/_helpers.py", line 134, in positional_wrapper
return wrapped(*args, **kwargs)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/http.py", line 1540, in execute
self._execute(http, self._order, self._requests)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/http.py", line 1470, in _execute
self._batch_uri, method="POST", body=body, headers=headers
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/google_auth_httplib2.py", line 201, in request
uri, method, body=body, headers=request_headers, **kwargs)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1994, in request
cachekey,
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1651, in _request
conn, request_uri, method, body, headers
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1589, in _conn_request
response = conn.getresponse()
File "/usr/lib64/python3.7/http/client.py", line 1369, in getresponse
response.begin()
File "/usr/lib64/python3.7/http/client.py", line 310, in begin
version, status, reason = self._read_status()
File "/usr/lib64/python3.7/http/client.py", line 271, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/lib64/python3.7/socket.py", line 589, in readinto
return self._sock.recv_into(b)
File "/usr/lib64/python3.7/ssl.py", line 1071, in recv_into
return self.read(nbytes, buffer)
File "/usr/lib64/python3.7/ssl.py", line 929, in read
return self._sslobj.read(len, buffer)
socket.timeout: The read operation timed out
File "/var/app/current/application/services/push.py", line 152, in execute_push
self.send_message(message=message)
File "/var/app/current/application/services/push.py", line 114, in send_message
response = messaging.send_multicast(message)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/firebase_admin/messaging.py", line 167, in send_multicast
return _get_messaging_service(app).send_all(messages, dry_run)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/firebase_admin/messaging.py", line 390, in send_all
batch.execute()
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/_helpers.py", line 134, in positional_wrapper
return wrapped(*args, **kwargs)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/http.py", line 1540, in execute
self._execute(http, self._order, self._requests)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/http.py", line 1470, in _execute
self._batch_uri, method="POST", body=body, headers=headers
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/google_auth_httplib2.py", line 201, in request
uri, method, body=body, headers=request_headers, **kwargs)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1994, in request
cachekey,
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1651, in _request
conn, request_uri, method, body, headers
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1589, in _conn_request
response = conn.getresponse()
File "/usr/lib64/python3.7/http/client.py", line 1369, in getresponse
response.begin()
File "/usr/lib64/python3.7/http/client.py", line 310, in begin
version, status, reason = self._read_status()
File "/usr/lib64/python3.7/http/client.py", line 271, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/lib64/python3.7/socket.py", line 589, in readinto
return self._sock.recv_into(b)
File "/usr/lib64/python3.7/ssl.py", line 1071, in recv_into
return self.read(nbytes, buffer)
File "/usr/lib64/python3.7/ssl.py", line 929, in read
return self._sslobj.read(len, buffer)
ssl.SSLError: [SSL: UNKNOWN_STATE] unknown state (_ssl.c:2570)
File "/var/app/current/application/services/push.py", line 114, in send_message
response = messaging.send_multicast(message)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/firebase_admin/messaging.py", line 167, in send_multicast
return _get_messaging_service(app).send_all(messages, dry_run)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/firebase_admin/messaging.py", line 390, in send_all
batch.execute()
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/_helpers.py", line 134, in positional_wrapper
return wrapped(*args, **kwargs)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/http.py", line 1540, in execute
self._execute(http, self._order, self._requests)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/http.py", line 1470, in _execute
self._batch_uri, method="POST", body=body, headers=headers
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/google_auth_httplib2.py", line 201, in request
uri, method, body=body, headers=request_headers, **kwargs)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1994, in request
cachekey,
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1651, in _request
conn, request_uri, method, body, headers
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1618, in _conn_request
content = response.read()
File "/usr/lib64/python3.7/http/client.py", line 468, in read
return self._readall_chunked()
File "/usr/lib64/python3.7/http/client.py", line 575, in _readall_chunked
chunk_left = self._get_chunk_left()
File "/usr/lib64/python3.7/http/client.py", line 563, in _get_chunk_left
self._read_and_discard_trailer()
File "/usr/lib64/python3.7/http/client.py", line 536, in _read_and_discard_trailer
line = self.fp.readline(_MAXLINE + 1)
AttributeError: 'NoneType' object has no attribute 'readline'
Some other exceptions are as follows:
ssl.SSLError: [SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:2570)
ssl.SSLError: [SSL: DECRYPTION_FAILED_OR_BAD_RECORD_MAC] decryption failed or bad record mac (_ssl.c:2570)
ssl.SSLError: [SSL: BLOCK_CIPHER_PAD_IS_WRONG] block cipher pad is wrong (_ssl.c:2570)
The code looks like this:
message = messaging.MulticastMessage(
tokens=tokens,
data=data,
notification=messaging.Notification(
title=title,
body=body
)
)
response = messaging.send_multicast(message)
This is quite strange for me since the tasks are failing and as a result, retrying. But somehow I am receiving multiple notifications. That means the notifications are being sent but the tasks are getting failed. Please help me out in understanding what could be the problem. Does it have something to do with the system date time?

ConnectionClosedError: Connection was closed before we received a valid response from endpoint URL:

So I'm trying to figure out Sagemaker and can get an endpoint for a tensorflow model up and running. Strangely enough, when I send small images to it, up to 220x220 (40kb), it runs just fine! However I get a connection closed error when I try anything larger, say 225x225.
I've tried from different networks and computers and get the same error. Also, I don't think I'm anywhere near the 5MB limit for sagemaker requests so I must admit I'm stumped on this one. Anyone happen to know what might be the cause of this? Thanks for your help.
I'm calling my model via boto as follows:
config = Config(connect_timeout=999999, read_timeout=9999999)
client = boto3.client("runtime.sagemaker", config=config)
data = {"instances": [{"x":x, "x2":x2}]}
response = client.invoke_endpoint(EndpointName='dcscn',
Body=json.dumps(data),
ContentType="application/json")
response_body = response['Body']
The error I get is:
Traceback (most recent call last):
File "/Users/bhhj/anaconda/lib/python3.6/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/Users/bhhj/anaconda/lib/python3.6/site-packages/urllib3/connectionpool.py", line 354, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/Users/bhhj/anaconda/lib/python3.6/http/client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/awsrequest.py", line 125, in _send_request
method, url, body, headers, *args, **kwargs)
File "/Users/bhhj/anaconda/lib/python3.6/http/client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/Users/bhhj/anaconda/lib/python3.6/http/client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/awsrequest.py", line 152, in _send_output
self.send(msg)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/awsrequest.py", line 236, in send
return super(AWSConnection, self).send(str)
File "/Users/bhhj/anaconda/lib/python3.6/http/client.py", line 986, in send
self.sock.sendall(data)
File "/Users/bhhj/anaconda/lib/python3.6/ssl.py", line 975, in sendall
v = self.send(byte_view[count:])
File "/Users/bhhj/anaconda/lib/python3.6/ssl.py", line 944, in send
return self._sslobj.write(data)
File "/Users/bhhj/anaconda/lib/python3.6/ssl.py", line 642, in write
return self._sslobj.write(data)
BrokenPipeError: [Errno 32] Broken pipe
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/httpsession.py", line 258, in send
decode_content=False,
File "/Users/bhhj/anaconda/lib/python3.6/site-packages/urllib3/connectionpool.py", line 638, in urlopen
_stacktrace=sys.exc_info()[2])
File "/Users/bhhj/anaconda/lib/python3.6/site-packages/urllib3/util/retry.py", line 343, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/Users/bhhj/anaconda/lib/python3.6/site-packages/urllib3/packages/six.py", line 685, in reraise
raise value.with_traceback(tb)
File "/Users/bhhj/anaconda/lib/python3.6/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/Users/bhhj/anaconda/lib/python3.6/site-packages/urllib3/connectionpool.py", line 354, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/Users/bhhj/anaconda/lib/python3.6/http/client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/awsrequest.py", line 125, in _send_request
method, url, body, headers, *args, **kwargs)
File "/Users/bhhj/anaconda/lib/python3.6/http/client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/Users/bhhj/anaconda/lib/python3.6/http/client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/awsrequest.py", line 152, in _send_output
self.send(msg)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/awsrequest.py", line 236, in send
return super(AWSConnection, self).send(str)
File "/Users/bhhj/anaconda/lib/python3.6/http/client.py", line 986, in send
self.sock.sendall(data)
File "/Users/bhhj/anaconda/lib/python3.6/ssl.py", line 975, in sendall
v = self.send(byte_view[count:])
File "/Users/bhhj/anaconda/lib/python3.6/ssl.py", line 944, in send
return self._sslobj.write(data)
File "/Users/bhhj/anaconda/lib/python3.6/ssl.py", line 642, in write
return self._sslobj.write(data)
urllib3.exceptions.ProtocolError: ('Connection aborted.', BrokenPipeError(32, 'Broken pipe'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "ping_endpoint.py", line 100, in <module>
ContentType="application/json")
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/client.py", line 357, in _api_call
return self._make_api_call(operation_name, kwargs)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/client.py", line 648, in _make_api_call
operation_model, request_dict, request_context)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/client.py", line 667, in _make_request
return self._endpoint.make_request(operation_model, request_dict)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/endpoint.py", line 102, in make_request
return self._send_request(request_dict, operation_model)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/endpoint.py", line 137, in _send_request
success_response, exception):
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/endpoint.py", line 231, in _needs_retry
caught_exception=caught_exception, request_dict=request_dict)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/hooks.py", line 356, in emit
return self._emitter.emit(aliased_event_name, **kwargs)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/hooks.py", line 228, in emit
return self._emit(event_name, kwargs)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/hooks.py", line 211, in _emit
response = handler(**kwargs)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/retryhandler.py", line 183, in __call__
if self._checker(attempts, response, caught_exception):
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/retryhandler.py", line 251, in __call__
caught_exception)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/retryhandler.py", line 277, in _should_retry
return self._checker(attempt_number, response, caught_exception)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/retryhandler.py", line 317, in __call__
caught_exception)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/retryhandler.py", line 223, in __call__
attempt_number, caught_exception)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/retryhandler.py", line 359, in _check_caught_exception
raise caught_exception
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/endpoint.py", line 200, in _do_get_response
http_response = self._send(request)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/endpoint.py", line 244, in _send
return self.http_session.send(request)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/httpsession.py", line 289, in send
endpoint_url=request.url
botocore.exceptions.ConnectionClosedError: Connection was closed before we received a valid response from endpoint URL:

It can be a network error.
If you are connected to a VPN , try after disconnecting it
try with better internet connection
my case, closing the VPN worked

In my case it was a firewall issue. How to troubleshoot: first, to do the same thing with aws cli and --debug flag. Already, with debug, you may see something. Then note the url and then try to ping the machine, in my case I had:
ping api.sagemaker.us-west-2.amazonaws.com
33 packets transmitted, 0 received, 100% packet loss, time 800ms.
Whitelisting the url solved the issue

Adding inf as float in a databank table with sqlalchemy

I have a dataframe which looks like this:
>data[i]
Property value
0 CO2 limit inf
So usually the value gets float numbers, but this time instead numbers, I set it up as inf, which translates to float('inf') in my model.
I would like to add this dataframe into a databank with using sqlalchemy.
I've created the table in databank via:
table = sa.Table(
table_name,
metadata,
sa.Column('id', sa.Integer, primary_key=True, autoincrement=True,
nullable=False),
sa.Column('Property', sa.String()),
sa.Column('value', sa.Float()),
schema=schema_name)
And I am adding the values via:
table.create()
data[i].to_sql('ubbb_'+i, engine, schema='model_draft',
if_exists='append', index=None)
At that point I am getting an error because I am trying to add inf in a column, which is defined as sa.Float, how can I fix this?
ERROR:
Traceback (most recent call last):
File "mimo.py", line 207, in <module>
a = conn.send_df(input_file_urbs)
File "/home/local/RL-INSTITUT/okan.akca/Schreibtisch/urbs-oemof/connection_oep/connection.py", line 63, in send_df
if not engine.dialect.has_table(engine, 'ubbb_'+i, 'model_draft'):
File "/usr/local/lib/python3.6/dist-packages/pandas/core/generic.py", line 2130, in to_sql
dtype=dtype)
File "/usr/local/lib/python3.6/dist-packages/pandas/io/sql.py", line 450, in to_sql
chunksize=chunksize, dtype=dtype)
File "/usr/local/lib/python3.6/dist-packages/pandas/io/sql.py", line 1127, in to_sql
table.insert(chunksize)
File "/usr/local/lib/python3.6/dist-packages/pandas/io/sql.py", line 641, in insert
self._execute_insert(conn, keys, chunk_iter)
File "/usr/local/lib/python3.6/dist-packages/pandas/io/sql.py", line 616, in _execute_insert
conn.execute(self.insert_statement(), data)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py", line 948, in execute
return meth(self, multiparams, params)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/sql/elements.py", line 269, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py", line 1060, in _execute_clauseelement
compiled_sql, distilled_params
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py", line 1200, in _execute_context
context)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py", line 1416, in _handle_dbapi_exception
util.reraise(*exc_info)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/util/compat.py", line 249, in reraise
raise value
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/base.py", line 1170, in _execute_context
context)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/dialects/postgresql/psycopg2.py", line 683, in do_executemany
cursor.executemany(statement, parameters)
File "/usr/local/lib/python3.6/dist-packages/oedialect/engine.py", line 303, in executemany
val = self.execute(query, p)
File "/usr/local/lib/python3.6/dist-packages/oedialect/engine.py", line 295, in execute
requires_connection_id=requires_connection_id)
File "/usr/local/lib/python3.6/dist-packages/oedialect/engine.py", line 312, in __execute_by_post
requires_connection_id=requires_connection_id)
File "/usr/local/lib/python3.6/dist-packages/oedialect/engine.py", line 212, in post
raise ConnectionException(json_response['reason'] if 'reason' in json_response else 'No reason returned')
oedialect.engine.ConnectionException: No reason returned
ERROR:sqlalchemy.pool.QueuePool:Exception during reset or similar
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/pool.py", line 709, in _finalize_fairy
fairy._reset(pool)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/pool.py", line 880, in _reset
pool._dialect.do_rollback(self)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/default.py", line 459, in do_rollback
dbapi_connection.rollback()
File "/usr/local/lib/python3.6/dist-packages/oedialect/engine.py", line 59, in rollback
requires_connection_id=True)
File "/usr/local/lib/python3.6/dist-packages/oedialect/engine.py", line 204, in post
headers=header, )
File "/usr/lib/python3/dist-packages/requests/api.py", line 112, in post
return request('post', url, data=data, json=json, **kwargs)
File "/usr/lib/python3/dist-packages/requests/api.py", line 58, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 520, in request
resp = self.send(prep, **send_kwargs)
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 630, in send
r = adapter.send(request, **kwargs)
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 440, in send
timeout=timeout
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 601, in urlopen
chunked=chunked)
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 346, in _make_request
self._validate_conn(conn)
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 852, in _validate_conn
conn.connect()
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 301, in connect
is_time_off = datetime.date.today() < RECENT_DATE
ImportError: sys.meta_path is None, Python is likely shutting down
ERROR:sqlalchemy.pool.QueuePool:Exception closing connection <oedialect.engine.OEConnection object at 0x7fb3cd220da0>
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/pool.py", line 709, in _finalize_fairy
fairy._reset(pool)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/pool.py", line 880, in _reset
pool._dialect.do_rollback(self)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/default.py", line 459, in do_rollback
dbapi_connection.rollback()
File "/usr/local/lib/python3.6/dist-packages/oedialect/engine.py", line 59, in rollback
requires_connection_id=True)
File "/usr/local/lib/python3.6/dist-packages/oedialect/engine.py", line 204, in post
headers=header, )
File "/usr/lib/python3/dist-packages/requests/api.py", line 112, in post
return request('post', url, data=data, json=json, **kwargs)
File "/usr/lib/python3/dist-packages/requests/api.py", line 58, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 520, in request
resp = self.send(prep, **send_kwargs)
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 630, in send
r = adapter.send(request, **kwargs)
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 440, in send
timeout=timeout
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 601, in urlopen
chunked=chunked)
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 346, in _make_request
self._validate_conn(conn)
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 852, in _validate_conn
conn.connect()
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 301, in connect
is_time_off = datetime.date.today() < RECENT_DATE
ImportError: sys.meta_path is None, Python is likely shutting down
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/pool.py", line 314, in _close_connection
self._dialect.do_close(connection)
File "/usr/local/lib/python3.6/dist-packages/sqlalchemy/engine/default.py", line 465, in do_close
dbapi_connection.close()
File "/usr/local/lib/python3.6/dist-packages/oedialect/engine.py", line 50, in close
requires_connection_id=True)
File "/usr/local/lib/python3.6/dist-packages/oedialect/engine.py", line 204, in post
headers=header, )
File "/usr/lib/python3/dist-packages/requests/api.py", line 112, in post
return request('post', url, data=data, json=json, **kwargs)
File "/usr/lib/python3/dist-packages/requests/api.py", line 58, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 520, in request
resp = self.send(prep, **send_kwargs)
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 630, in send
r = adapter.send(request, **kwargs)
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 440, in send
timeout=timeout
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 601, in urlopen
chunked=chunked)
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 346, in _make_request
self._validate_conn(conn)
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 852, in _validate_conn
conn.connect()
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 301, in connect
is_time_off = datetime.date.today() < RECENT_DATE
ImportError: sys.meta_path is None, Python is likely shutting down