Just to give a brief introduction, I am using elastic beanstalk from Amazon to deploy my python (Falcon) application. I have two environments of elastic beanstalk (one being the main app server and the other one a worker environment). The worker environment receives HTTP requests for async tasks and then executes them. Most of the tasks are either push notifications (mobile devices) or sending emails. The strange part is that my tasks are failing with errors that I am unable to determine the cause for (this works fine for my local environment). Some of the tracebacks look like this:
self.send_message(message=message)
File "/var/app/current/application/services/push.py", line 114, in send_message
response = messaging.send_multicast(message)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/firebase_admin/messaging.py", line 167, in send_multicast
return _get_messaging_service(app).send_all(messages, dry_run)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/firebase_admin/messaging.py", line 390, in send_all
batch.execute()
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/_helpers.py", line 134, in positional_wrapper
return wrapped(*args, **kwargs)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/http.py", line 1540, in execute
self._execute(http, self._order, self._requests)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/http.py", line 1470, in _execute
self._batch_uri, method="POST", body=body, headers=headers
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/google_auth_httplib2.py", line 201, in request
uri, method, body=body, headers=request_headers, **kwargs)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1994, in request
cachekey,
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1651, in _request
conn, request_uri, method, body, headers
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1589, in _conn_request
response = conn.getresponse()
File "/usr/lib64/python3.7/http/client.py", line 1369, in getresponse
response.begin()
File "/usr/lib64/python3.7/http/client.py", line 310, in begin
version, status, reason = self._read_status()
File "/usr/lib64/python3.7/http/client.py", line 271, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/lib64/python3.7/socket.py", line 589, in readinto
return self._sock.recv_into(b)
File "/usr/lib64/python3.7/ssl.py", line 1071, in recv_into
return self.read(nbytes, buffer)
File "/usr/lib64/python3.7/ssl.py", line 929, in read
return self._sslobj.read(len, buffer)
socket.timeout: The read operation timed out
File "/var/app/current/application/services/push.py", line 152, in execute_push
self.send_message(message=message)
File "/var/app/current/application/services/push.py", line 114, in send_message
response = messaging.send_multicast(message)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/firebase_admin/messaging.py", line 167, in send_multicast
return _get_messaging_service(app).send_all(messages, dry_run)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/firebase_admin/messaging.py", line 390, in send_all
batch.execute()
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/_helpers.py", line 134, in positional_wrapper
return wrapped(*args, **kwargs)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/http.py", line 1540, in execute
self._execute(http, self._order, self._requests)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/http.py", line 1470, in _execute
self._batch_uri, method="POST", body=body, headers=headers
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/google_auth_httplib2.py", line 201, in request
uri, method, body=body, headers=request_headers, **kwargs)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1994, in request
cachekey,
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1651, in _request
conn, request_uri, method, body, headers
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1589, in _conn_request
response = conn.getresponse()
File "/usr/lib64/python3.7/http/client.py", line 1369, in getresponse
response.begin()
File "/usr/lib64/python3.7/http/client.py", line 310, in begin
version, status, reason = self._read_status()
File "/usr/lib64/python3.7/http/client.py", line 271, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/lib64/python3.7/socket.py", line 589, in readinto
return self._sock.recv_into(b)
File "/usr/lib64/python3.7/ssl.py", line 1071, in recv_into
return self.read(nbytes, buffer)
File "/usr/lib64/python3.7/ssl.py", line 929, in read
return self._sslobj.read(len, buffer)
ssl.SSLError: [SSL: UNKNOWN_STATE] unknown state (_ssl.c:2570)
File "/var/app/current/application/services/push.py", line 114, in send_message
response = messaging.send_multicast(message)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/firebase_admin/messaging.py", line 167, in send_multicast
return _get_messaging_service(app).send_all(messages, dry_run)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/firebase_admin/messaging.py", line 390, in send_all
batch.execute()
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/_helpers.py", line 134, in positional_wrapper
return wrapped(*args, **kwargs)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/http.py", line 1540, in execute
self._execute(http, self._order, self._requests)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/http.py", line 1470, in _execute
self._batch_uri, method="POST", body=body, headers=headers
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/google_auth_httplib2.py", line 201, in request
uri, method, body=body, headers=request_headers, **kwargs)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1994, in request
cachekey,
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1651, in _request
conn, request_uri, method, body, headers
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1618, in _conn_request
content = response.read()
File "/usr/lib64/python3.7/http/client.py", line 468, in read
return self._readall_chunked()
File "/usr/lib64/python3.7/http/client.py", line 575, in _readall_chunked
chunk_left = self._get_chunk_left()
File "/usr/lib64/python3.7/http/client.py", line 563, in _get_chunk_left
self._read_and_discard_trailer()
File "/usr/lib64/python3.7/http/client.py", line 536, in _read_and_discard_trailer
line = self.fp.readline(_MAXLINE + 1)
AttributeError: 'NoneType' object has no attribute 'readline'
Some other exceptions are as follows:
ssl.SSLError: [SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:2570)
ssl.SSLError: [SSL: DECRYPTION_FAILED_OR_BAD_RECORD_MAC] decryption failed or bad record mac (_ssl.c:2570)
ssl.SSLError: [SSL: BLOCK_CIPHER_PAD_IS_WRONG] block cipher pad is wrong (_ssl.c:2570)
The code looks like this:
message = messaging.MulticastMessage(
tokens=tokens,
data=data,
notification=messaging.Notification(
title=title,
body=body
)
)
response = messaging.send_multicast(message)
This is quite strange for me since the tasks are failing and as a result, retrying. But somehow I am receiving multiple notifications. That means the notifications are being sent but the tasks are getting failed. Please help me out in understanding what could be the problem. Does it have something to do with the system date time?
Related
I create a Django App and send rest http request to Plaid. if I start up django alone(python manage.py run server), it works fine. But if I use Nginx + Gunicorn + Django, I will get error. The error message is:
File "/var/www/sp_plaid/api/views.py", line 91, in create_link_token
response = client.link_token_create(p_request)
File "/usr/local/lib/python3.9/dist-packages/plaid/api_client.py", line 769, in __call__
return self.callable(self, *args, **kwargs)
File "/usr/local/lib/python3.9/dist-packages/plaid/api/plaid_api.py", line 6863, in __link_token_create
return self.call_with_http_info(**kwargs)
File "/usr/local/lib/python3.9/dist-packages/plaid/api_client.py", line 831, in call_with_http_info
return self.api_client.call_api(
File "/usr/local/lib/python3.9/dist-packages/plaid/api_client.py", line 406, in call_api
return self.__call_api(resource_path, method,
File "/usr/local/lib/python3.9/dist-packages/plaid/api_client.py", line 193, in __call_api
response_data = self.request(
File "/usr/local/lib/python3.9/dist-packages/plaid/api_client.py", line 452, in request
return self.rest_client.POST(url,
File "/usr/local/lib/python3.9/dist-packages/plaid/rest.py", line 264, in POST
return self.request("POST", url,
File "/usr/local/lib/python3.9/dist-packages/plaid/rest.py", line 150, in request
r = self.pool_manager.request(
File "/usr/lib/python3/dist-packages/urllib3/request.py", line 78, in request
return self.request_encode_body(
File "/usr/lib/python3/dist-packages/urllib3/request.py", line 170, in request_encode_body
return self.urlopen(method, url, **extra_kw)
File "/usr/lib/python3/dist-packages/urllib3/poolmanager.py", line 375, in urlopen
response = conn.urlopen(method, u.request_uri, **kw)
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 394, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 234, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "/usr/lib/python3.9/http/client.py", line 1279, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.9/http/client.py", line 1320, in _send_request
self.putheader(hdr, value)
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 219, in putheader
_
HTTPConnection.putheader(self, header, *values)
File "/usr/lib/python3.9/http/client.py", line 1256, in putheader
if _is_illegal_header_value(values):
TypeError: expected string or bytes-like object
python version: 3.8, Django:3.2.4.
Anyone can help? Thanks.
Do you know why I have this error ?
timeout: The read operation timed out
How to except it please ? From which package can I retrieve the tiemout exception ?
File "C:\Users\root\Desktop\bot\sheetapi.py", line 58, in
update
spreadsheetId=SAMPLE_SPREADSHEET_ID, range=SAMPLE_RANGE_NAME, valueInputOption='USER_ENTERED', body=body).execute()
File
"C:\Users\root\Anaconda3\lib\site-packages\googleapiclient_helpers.py",
line 134, in positional_wrapper
return wrapped(*args, **kwargs)
File
"C:\Users\root\Anaconda3\lib\site-packages\googleapiclient\http.py",
line 892, in execute
headers=self.headers,
File
"C:\Users\root\Anaconda3\lib\site-packages\googleapiclient\http.py",
line 195, in _retry_request
raise exception
File
"C:\Users\root\Anaconda3\lib\site-packages\googleapiclient\http.py",
line 172, in _retry_request
resp, content = http.request(uri, method, *args, **kwargs)
File
"C:\Users\root\Anaconda3\lib\site-packages\google_auth_httplib2.py",
line 198, in request
uri, method, body=body, headers=request_headers, **kwargs)
File
"C:\Users\root\Anaconda3\lib\site-packages\httplib2__init__.py", line
1514, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File
"C:\Users\root\Anaconda3\lib\site-packages\httplib2__init__.py", line
1264, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File
"C:\Users\root\Anaconda3\lib\site-packages\httplib2__init__.py", line
1217, in _conn_request
response = conn.getresponse()
File "C:\Users\root\Anaconda3\lib\http\client.py", line 1331, in
getresponse
response.begin()
File "C:\Users\root\Anaconda3\lib\http\client.py", line 297, in
begin
version, status, reason = self._read_status()
File "C:\Users\root\Anaconda3\lib\http\client.py", line 258, in
_read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "C:\Users\root\Anaconda3\lib\socket.py", line 586, in readinto
return self._sock.recv_into(b)
File "C:\Users\root\Anaconda3\lib\ssl.py", line 1009, in recv_into
return self.read(nbytes, buffer)
File "C:\Users\root\Anaconda3\lib\ssl.py", line 871, in read
return self._sslobj.read(len, buffer)
File "C:\Users\root\Anaconda3\lib\ssl.py", line 631, in read
v = self._sslobj.read(len, buffer)
timeout: The read operation timed out
I am trying to deploy Rest Api with Chalice.
When I do chalice deploy.
The code worked with chalice local. But it fails in Deployment.
I get the following error:
Creating deployment package.
Updating policy for IAM role: learnChalice-dev
Updating lambda function: learnChalice-dev
Traceback (most recent call last):
File "/home/dragon/.local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/home/dragon/.local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 354, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/http/client.py", line 1252, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/awsrequest.py", line 92, in _send_request
method, url, body, headers, *args, **kwargs)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/http/client.py", line 1298, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/http/client.py", line 1247, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/awsrequest.py", line 119, in _send_output
self.send(msg)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/awsrequest.py", line 203, in send
return super(AWSConnection, self).send(str)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/http/client.py", line 987, in send
self.sock.sendall(data)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/ssl.py", line 1034, in sendall
v = self.send(byte_view[count:])
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/ssl.py", line 1003, in send
return self._sslobj.write(data)
socket.timeout: The write operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/httpsession.py", line 263, in send
chunked=self._chunked(request.headers),
File "/home/dragon/.local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 638, in urlopen
_stacktrace=sys.exc_info()[2])
File "/home/dragon/.local/lib/python3.7/site-packages/urllib3/util/retry.py", line 344, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/home/dragon/.local/lib/python3.7/site-packages/urllib3/packages/six.py", line 685, in reraise
raise value.with_traceback(tb)
File "/home/dragon/.local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/home/dragon/.local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 354, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/http/client.py", line 1252, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/awsrequest.py", line 92, in _send_request
method, url, body, headers, *args, **kwargs)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/http/client.py", line 1298, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/http/client.py", line 1247, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/awsrequest.py", line 119, in _send_output
self.send(msg)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/awsrequest.py", line 203, in send
return super(AWSConnection, self).send(str)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/http/client.py", line 987, in send
self.sock.sendall(data)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/ssl.py", line 1034, in sendall
v = self.send(byte_view[count:])
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/ssl.py", line 1003, in send
return self._sslobj.write(data)
urllib3.exceptions.ProtocolError: ('Connection aborted.', timeout('The write operation timed out'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/chalice/cli/__init__.py", line 599, in main
return cli(obj={})
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/click/core.py", line 829, in __call__
return self.main(*args, **kwargs)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/click/core.py", line 782, in main
rv = self.invoke(ctx)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/click/core.py", line 1259, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/click/core.py", line 1066, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/click/core.py", line 610, in invoke
return callback(*args, **kwargs)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/click/decorators.py", line 21, in new_func
return f(get_current_context(), *args, **kwargs)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/chalice/cli/init.py", line 206, in deploy
deployed_values = d.deploy(config, chalice_stage_name=stage)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/chalice/deploy/deployer.py", line 353, in deploy
return self._deploy(config, chalice_stage_name)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/chalice/deploy/deployer.py", line 366, in _deploy
self._executor.execute(plan)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/chalice/deploy/executor.py", line 44, in execute
self._default_handler)(instruction)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/chalice/deploy/executor.py", line 56, in _do_apicall
result = method(**final_kwargs)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/chalice/awsclient.py", line 273, in update_function
zip_contents=zip_contents)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/chalice/awsclient.py", line 294, in _update_function_code
FunctionName=function_name, ZipFile=zip_contents)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/client.py", line 316, in _api_call
return self._make_api_call(operation_name, kwargs)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/client.py", line 613, in _make_api_call
operation_model, request_dict, request_context)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/client.py", line 632, in _make_request
return self._endpoint.make_request(operation_model, request_dict)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/endpoint.py", line 102, in make_request
return self._send_request(request_dict, operation_model)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/endpoint.py", line 137, in _send_request
success_response, exception):
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/endpoint.py", line 231, in _needs_retry
caught_exception=caught_exception, request_dict=request_dict)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/hooks.py", line 356, in emit
return self._emitter.emit(aliased_event_name, **kwargs)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/hooks.py", line 228, in emit
return self._emit(event_name, kwargs)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/hooks.py", line 211, in _emit
response = handler(**kwargs)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/retryhandler.py", line 183, in __call__
if self._checker(attempts, response, caught_exception):
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/retryhandler.py", line 251, in __call__
caught_exception)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/retryhandler.py", line 277, in _should_retry
return self._checker(attempt_number, response, caught_exception)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/retryhandler.py", line 317, in __call__
caught_exception)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/retryhandler.py", line 223, in __call__
attempt_number, caught_exception)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/retryhandler.py", line 359, in _check_caught_exception
raise caught_exception
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/endpoint.py", line 200, in _do_get_response
http_response = self._send(request)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/endpoint.py", line 244, in _send
return self.http_session.send(request)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/httpsession.py", line 294, in send
endpoint_url=request.url
botocore.exceptions.ConnectionClosedError: Connection was closed before we received a valid response from endpoint URL: "https://lambda.ap-south-1.amazonaws.com/2015-03-31/functions/learnChalice-dev/code".
Thanks a lot for help.
Based on the comments, the solution was to increase default timeout:
chalice deploy --connection-timeout 300
We have a kubernetes cron job on GCP that submits several copies of the same Python dataflow job, each in its own container. Whenever we need a new copy of the job, we just add it to the spec->jobTemplate->spec->template->spec->containers part of the cron job yaml and adjust the dataflow job parameters as needed. This usually works fine, but the latest copy we tried to add does not work. The existing copies are all still working as expected. The job seems to fail on submission to GCP, and the error message is not very helpful:
Traceback (most recent call last):
File "/app/job.py", line 117, in <module>
newness.pipeline.run_dataflow(sys.argv)
File "/app/newness/pipeline.py", line 480, in run_dataflow
result = pipe.run()
File "/usr/local/lib/python2.7/dist-packages/apache_beam/pipeline.py", line 403, in run
self.to_runner_api(), self.runner, self._options).run(False)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/pipeline.py", line 416, in run
return self.runner.run_pipeline(self)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/dataflow/dataflow_runner.py", line 389, in run_pipeline
self.dataflow_client.create_job(self.job), self)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/utils/retry.py", line 184, in wrapper
return fun(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/dataflow/internal/apiclient.py", line 504, in create_job
return self.submit_job_description(job)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/utils/retry.py", line 184, in wrapper
return fun(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/dataflow/internal/apiclient.py", line 551, in submit_job_description
response = self._client.projects_locations_jobs.Create(request)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py", line 578, in Create
config, request, global_params=global_params)
File "/usr/local/lib/python2.7/dist-packages/apitools/base/py/base_api.py", line 731, in _RunMethod
return self.ProcessHttpResponse(method_config, http_response, request)
File "/usr/local/lib/python2.7/dist-packages/apitools/base/py/base_api.py", line 737, in ProcessHttpResponse
self.__ProcessHttpResponse(method_config, http_response, request))
File "/usr/local/lib/python2.7/dist-packages/apitools/base/py/base_api.py", line 604, in __ProcessHttpResponse
http_response, method_config=method_config, request=request)
apitools.base.py.exceptions.HttpError: <exception str() failed>
The job does not appear in the dataflow console at all.
The previous lines of the container logs look like:
2019-10-13T03:57:47.725542287Z Successfully downloaded apache-beam
2019-10-13T03:58:17.125601280Z INFO:root:Staging SDK sources from PyPI to gs://gcs-bucket-name/staging/newness-boosting-c2898.1570936519.827087/dataflow_python_sdk.tar
2019-10-13T03:58:17.324843623Z INFO:root:Starting GCS upload to gs://gcs-bucket-name/staging/newness-boosting-c2898.1570936519.827087/dataflow_python_sdk.tar...
2019-10-13T03:58:24.825657227Z INFO:root:Completed GCS upload to gs://gcs-bucket-name/staging/newness-boosting-c2898.1570936519.827087/dataflow_python_sdk.tar
2019-10-13T03:58:25.225646529Z INFO:root:Downloading binary distribtution of the SDK from PyPi
2019-10-13T03:58:25.225716554Z INFO:root:Executing command: ['/usr/bin/python', '-m', 'pip', 'download', '--dest', '/tmp/tmpk5TfMS', 'apache-beam==2.8.0', '--no-deps', '--only-binary', ':all:', '--python-version', '27', '--implementation', 'cp', '--abi', 'cp27mu', '--platform', 'manylinux1_x86_64']
2019-10-13T03:59:33.926186906Z Collecting apache-beam==2.8.0
2019-10-13T03:59:52.125678183Z Using cached https://files.pythonhosted.org/packages/0f/63/ea5453ba656d060936acf41d2ec057f23aafd69649e2129ac66fdda67d48/apache_beam-2.8.0-cp27-cp27mu-manylinux1_x86_64.whl
2019-10-13T04:00:11.525435891Z Saved /tmp/tmpk5TfMS/apache_beam-2.8.0-cp27-cp27mu-manylinux1_x86_64.whl
2019-10-13T04:00:12.025054706Z Successfully downloaded apache-beam
2019-10-13T04:00:26.726190542Z INFO:root:Staging binary distribution of the SDK from PyPI to gs://gcs-bucket-name/staging/newness-boosting-c2898.1570936519.827087/apache_beam-2.8.0-cp27-cp27mu-manylinux1_x86_64.whl
2019-10-13T04:00:26.825618945Z INFO:root:Starting GCS upload to gs://gcs-bucket-name/staging/newness-boosting-c2898.1570936519.827087/apache_beam-2.8.0-cp27-cp27mu-manylinux1_x86_64.whl...
2019-10-13T04:00:33.725522899Z INFO:root:Completed GCS upload to gs://gcs-bucket-name/staging/newness-boosting-c2898.1570936519.827087/apache_beam-2.8.0-cp27-cp27mu-manylinux1_x86_64.whl
2019-10-13T04:06:14.525017097Z Traceback (most recent call last):
...
Why is this job failing to submit? Are there any other logs we can look at to see the cause of this failure?
(Most of our dataflow jobs are written in Java, where the error messages are usually more helpful.)
UPDATE: Running job locally (Windows) with apache-beam 2.16 has the same issue but more logging detail:
...
INFO:root:Starting GCS upload to gs://gcs-bucket-name/staging/newness-boosting-c2898.1571606418.971000/apache_beam-2.16.0-cp27-cp27mu-manylinux1_x86_64.whl...
INFO:root:Completed GCS upload to gs://gcs-bucket-name/staging/newness-boosting-c2898.1571606418.971000/apache_beam-2.16.0-cp27-cp27mu-manylinux1_x86_64.whl in 3 seconds.
WARNING:root:Discarding unparseable args: ['job.py', '--days_history=30']
WARNING:root:Discarding unparseable args: ['job.py', '--days_history=30']
WARNING:root:Retry with exponential backoff: waiting for 2.64795143823 seconds before retrying submit_job_description because we caught exception: error: [Errno 10053] An established connection was aborted by the software in your host machine
Traceback for above exception (most recent call last):
File "C:\Python27\lib\site-packages\apache_beam\utils\retry.py", line 206, in wrapper
return fun(*args, **kwargs)
File "C:\Python27\lib\site-packages\apache_beam\runners\dataflow\internal\apiclient.py", line 593, in submit_job_description
response = self._client.projects_locations_jobs.Create(request)
File "C:\Python27\lib\site-packages\apache_beam\runners\dataflow\internal\clients\dataflow\dataflow_v1b3_client.py", line 657, in Create
config, request, global_params=global_params)
File "C:\Python27\lib\site-packages\apitools\base\py\base_api.py", line 729, in _RunMethod
http, http_request, **opts)
File "C:\Python27\lib\site-packages\apitools\base\py\http_wrapper.py", line 346, in MakeRequest
check_response_func=check_response_func)
File "C:\Python27\lib\site-packages\apitools\base\py\http_wrapper.py", line 396, in _MakeRequestNoRetry
redirections=redirections, connection_type=connection_type)
File "C:\Python27\lib\site-packages\oauth2client\transport.py", line 169, in new_request
redirections, connection_type)
File "C:\Python27\lib\site-packages\oauth2client\transport.py", line 169, in new_request
redirections, connection_type)
File "C:\Python27\lib\site-packages\httplib2\__init__.py", line 1694, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "C:\Python27\lib\site-packages\httplib2\__init__.py", line 1434, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "C:\Python27\lib\site-packages\httplib2\__init__.py", line 1390, in _conn_request
response = conn.getresponse()
File "C:\Python27\lib\httplib.py", line 1121, in getresponse
response.begin()
File "C:\Python27\lib\httplib.py", line 438, in begin
version, status, reason = self._read_status()
File "C:\Python27\lib\httplib.py", line 394, in _read_status
line = self.fp.readline(_MAXLINE + 1)
File "C:\Python27\lib\socket.py", line 480, in readline
data = self._sock.recv(self._rbufsize)
File "C:\Python27\lib\ssl.py", line 754, in recv
return self.read(buflen)
File "C:\Python27\lib\ssl.py", line 641, in read
v = self._sslobj.read(len)
... retries 4 times total ...
Traceback (most recent call last):
File "job.py", line 117, in <module>
newness.pipeline.run_dataflow(sys.argv)
File "C:\Users\LeeW\Desktop\newness\newness\pipeline.py", line 480, in run_dataflow
result = pipe.run()
File "C:\Python27\lib\site-packages\apache_beam\pipeline.py", line 407, in run
self._options).run(False)
File "C:\Python27\lib\site-packages\apache_beam\pipeline.py", line 420, in run
return self.runner.run_pipeline(self, self._options)
File "C:\Python27\lib\site-packages\apache_beam\runners\dataflow\dataflow_runner.py", line 485, in run_pipeline
self.dataflow_client.create_job(self.job), self)
File "C:\Python27\lib\site-packages\apache_beam\utils\retry.py", line 206, in wrapper
return fun(*args, **kwargs)
File "C:\Python27\lib\site-packages\apache_beam\runners\dataflow\internal\apiclient.py", line 546, in create_job
return self.submit_job_description(job)
File "C:\Python27\lib\site-packages\apache_beam\utils\retry.py", line 219, in wrapper
raise_with_traceback(exn, exn_traceback)
File "C:\Python27\lib\site-packages\apache_beam\utils\retry.py", line 206, in wrapper
return fun(*args, **kwargs)
File "C:\Python27\lib\site-packages\apache_beam\runners\dataflow\internal\apiclient.py", line 593, in submit_job_description
response = self._client.projects_locations_jobs.Create(request)
File "C:\Python27\lib\site-packages\apache_beam\runners\dataflow\internal\clients\dataflow\dataflow_v1b3_client.py", line 657, in Create
config, request, global_params=global_params)
File "C:\Python27\lib\site-packages\apitools\base\py\base_api.py", line 729, in _RunMethod
http, http_request, **opts)
File "C:\Python27\lib\site-packages\apitools\base\py\http_wrapper.py", line 346, in MakeRequest
check_response_func=check_response_func)
File "C:\Python27\lib\site-packages\apitools\base\py\http_wrapper.py", line 396, in _MakeRequestNoRetry
redirections=redirections, connection_type=connection_type)
File "C:\Python27\lib\site-packages\oauth2client\transport.py", line 169, in new_request
redirections, connection_type)
File "C:\Python27\lib\site-packages\oauth2client\transport.py", line 169, in new_request
redirections, connection_type)
File "C:\Python27\lib\site-packages\httplib2\__init__.py", line 1694, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "C:\Python27\lib\site-packages\httplib2\__init__.py", line 1434, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "C:\Python27\lib\site-packages\httplib2\__init__.py", line 1390, in _conn_request
response = conn.getresponse()
File "C:\Python27\lib\httplib.py", line 1121, in getresponse
response.begin()
File "C:\Python27\lib\httplib.py", line 438, in begin
version, status, reason = self._read_status()
File "C:\Python27\lib\httplib.py", line 394, in _read_status
line = self.fp.readline(_MAXLINE + 1)
File "C:\Python27\lib\socket.py", line 480, in readline
data = self._sock.recv(self._rbufsize)
File "C:\Python27\lib\ssl.py", line 754, in recv
return self.read(buflen)
File "C:\Python27\lib\ssl.py", line 641, in read
v = self._sslobj.read(len)
socket.error: [Errno 10053] An established connection was aborted by the software in your host machine
Which version of Beam Python SDK are you using?
So I'm trying to figure out Sagemaker and can get an endpoint for a tensorflow model up and running. Strangely enough, when I send small images to it, up to 220x220 (40kb), it runs just fine! However I get a connection closed error when I try anything larger, say 225x225.
I've tried from different networks and computers and get the same error. Also, I don't think I'm anywhere near the 5MB limit for sagemaker requests so I must admit I'm stumped on this one. Anyone happen to know what might be the cause of this? Thanks for your help.
I'm calling my model via boto as follows:
config = Config(connect_timeout=999999, read_timeout=9999999)
client = boto3.client("runtime.sagemaker", config=config)
data = {"instances": [{"x":x, "x2":x2}]}
response = client.invoke_endpoint(EndpointName='dcscn',
Body=json.dumps(data),
ContentType="application/json")
response_body = response['Body']
The error I get is:
Traceback (most recent call last):
File "/Users/bhhj/anaconda/lib/python3.6/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/Users/bhhj/anaconda/lib/python3.6/site-packages/urllib3/connectionpool.py", line 354, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/Users/bhhj/anaconda/lib/python3.6/http/client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/awsrequest.py", line 125, in _send_request
method, url, body, headers, *args, **kwargs)
File "/Users/bhhj/anaconda/lib/python3.6/http/client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/Users/bhhj/anaconda/lib/python3.6/http/client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/awsrequest.py", line 152, in _send_output
self.send(msg)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/awsrequest.py", line 236, in send
return super(AWSConnection, self).send(str)
File "/Users/bhhj/anaconda/lib/python3.6/http/client.py", line 986, in send
self.sock.sendall(data)
File "/Users/bhhj/anaconda/lib/python3.6/ssl.py", line 975, in sendall
v = self.send(byte_view[count:])
File "/Users/bhhj/anaconda/lib/python3.6/ssl.py", line 944, in send
return self._sslobj.write(data)
File "/Users/bhhj/anaconda/lib/python3.6/ssl.py", line 642, in write
return self._sslobj.write(data)
BrokenPipeError: [Errno 32] Broken pipe
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/httpsession.py", line 258, in send
decode_content=False,
File "/Users/bhhj/anaconda/lib/python3.6/site-packages/urllib3/connectionpool.py", line 638, in urlopen
_stacktrace=sys.exc_info()[2])
File "/Users/bhhj/anaconda/lib/python3.6/site-packages/urllib3/util/retry.py", line 343, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/Users/bhhj/anaconda/lib/python3.6/site-packages/urllib3/packages/six.py", line 685, in reraise
raise value.with_traceback(tb)
File "/Users/bhhj/anaconda/lib/python3.6/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/Users/bhhj/anaconda/lib/python3.6/site-packages/urllib3/connectionpool.py", line 354, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/Users/bhhj/anaconda/lib/python3.6/http/client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/awsrequest.py", line 125, in _send_request
method, url, body, headers, *args, **kwargs)
File "/Users/bhhj/anaconda/lib/python3.6/http/client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/Users/bhhj/anaconda/lib/python3.6/http/client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/awsrequest.py", line 152, in _send_output
self.send(msg)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/awsrequest.py", line 236, in send
return super(AWSConnection, self).send(str)
File "/Users/bhhj/anaconda/lib/python3.6/http/client.py", line 986, in send
self.sock.sendall(data)
File "/Users/bhhj/anaconda/lib/python3.6/ssl.py", line 975, in sendall
v = self.send(byte_view[count:])
File "/Users/bhhj/anaconda/lib/python3.6/ssl.py", line 944, in send
return self._sslobj.write(data)
File "/Users/bhhj/anaconda/lib/python3.6/ssl.py", line 642, in write
return self._sslobj.write(data)
urllib3.exceptions.ProtocolError: ('Connection aborted.', BrokenPipeError(32, 'Broken pipe'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "ping_endpoint.py", line 100, in <module>
ContentType="application/json")
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/client.py", line 357, in _api_call
return self._make_api_call(operation_name, kwargs)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/client.py", line 648, in _make_api_call
operation_model, request_dict, request_context)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/client.py", line 667, in _make_request
return self._endpoint.make_request(operation_model, request_dict)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/endpoint.py", line 102, in make_request
return self._send_request(request_dict, operation_model)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/endpoint.py", line 137, in _send_request
success_response, exception):
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/endpoint.py", line 231, in _needs_retry
caught_exception=caught_exception, request_dict=request_dict)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/hooks.py", line 356, in emit
return self._emitter.emit(aliased_event_name, **kwargs)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/hooks.py", line 228, in emit
return self._emit(event_name, kwargs)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/hooks.py", line 211, in _emit
response = handler(**kwargs)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/retryhandler.py", line 183, in __call__
if self._checker(attempts, response, caught_exception):
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/retryhandler.py", line 251, in __call__
caught_exception)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/retryhandler.py", line 277, in _should_retry
return self._checker(attempt_number, response, caught_exception)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/retryhandler.py", line 317, in __call__
caught_exception)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/retryhandler.py", line 223, in __call__
attempt_number, caught_exception)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/retryhandler.py", line 359, in _check_caught_exception
raise caught_exception
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/endpoint.py", line 200, in _do_get_response
http_response = self._send(request)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/endpoint.py", line 244, in _send
return self.http_session.send(request)
File "/Users/bhhj/.local/lib/python3.6/site-packages/botocore/httpsession.py", line 289, in send
endpoint_url=request.url
botocore.exceptions.ConnectionClosedError: Connection was closed before we received a valid response from endpoint URL:
It can be a network error.
If you are connected to a VPN , try after disconnecting it
try with better internet connection
my case, closing the VPN worked
In my case it was a firewall issue. How to troubleshoot: first, to do the same thing with aws cli and --debug flag. Already, with debug, you may see something. Then note the url and then try to ping the machine, in my case I had:
ping api.sagemaker.us-west-2.amazonaws.com
33 packets transmitted, 0 received, 100% packet loss, time 800ms.
Whitelisting the url solved the issue