Related
Just to give a brief introduction, I am using elastic beanstalk from Amazon to deploy my python (Falcon) application. I have two environments of elastic beanstalk (one being the main app server and the other one a worker environment). The worker environment receives HTTP requests for async tasks and then executes them. Most of the tasks are either push notifications (mobile devices) or sending emails. The strange part is that my tasks are failing with errors that I am unable to determine the cause for (this works fine for my local environment). Some of the tracebacks look like this:
self.send_message(message=message)
File "/var/app/current/application/services/push.py", line 114, in send_message
response = messaging.send_multicast(message)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/firebase_admin/messaging.py", line 167, in send_multicast
return _get_messaging_service(app).send_all(messages, dry_run)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/firebase_admin/messaging.py", line 390, in send_all
batch.execute()
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/_helpers.py", line 134, in positional_wrapper
return wrapped(*args, **kwargs)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/http.py", line 1540, in execute
self._execute(http, self._order, self._requests)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/http.py", line 1470, in _execute
self._batch_uri, method="POST", body=body, headers=headers
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/google_auth_httplib2.py", line 201, in request
uri, method, body=body, headers=request_headers, **kwargs)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1994, in request
cachekey,
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1651, in _request
conn, request_uri, method, body, headers
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1589, in _conn_request
response = conn.getresponse()
File "/usr/lib64/python3.7/http/client.py", line 1369, in getresponse
response.begin()
File "/usr/lib64/python3.7/http/client.py", line 310, in begin
version, status, reason = self._read_status()
File "/usr/lib64/python3.7/http/client.py", line 271, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/lib64/python3.7/socket.py", line 589, in readinto
return self._sock.recv_into(b)
File "/usr/lib64/python3.7/ssl.py", line 1071, in recv_into
return self.read(nbytes, buffer)
File "/usr/lib64/python3.7/ssl.py", line 929, in read
return self._sslobj.read(len, buffer)
socket.timeout: The read operation timed out
File "/var/app/current/application/services/push.py", line 152, in execute_push
self.send_message(message=message)
File "/var/app/current/application/services/push.py", line 114, in send_message
response = messaging.send_multicast(message)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/firebase_admin/messaging.py", line 167, in send_multicast
return _get_messaging_service(app).send_all(messages, dry_run)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/firebase_admin/messaging.py", line 390, in send_all
batch.execute()
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/_helpers.py", line 134, in positional_wrapper
return wrapped(*args, **kwargs)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/http.py", line 1540, in execute
self._execute(http, self._order, self._requests)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/http.py", line 1470, in _execute
self._batch_uri, method="POST", body=body, headers=headers
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/google_auth_httplib2.py", line 201, in request
uri, method, body=body, headers=request_headers, **kwargs)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1994, in request
cachekey,
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1651, in _request
conn, request_uri, method, body, headers
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1589, in _conn_request
response = conn.getresponse()
File "/usr/lib64/python3.7/http/client.py", line 1369, in getresponse
response.begin()
File "/usr/lib64/python3.7/http/client.py", line 310, in begin
version, status, reason = self._read_status()
File "/usr/lib64/python3.7/http/client.py", line 271, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/lib64/python3.7/socket.py", line 589, in readinto
return self._sock.recv_into(b)
File "/usr/lib64/python3.7/ssl.py", line 1071, in recv_into
return self.read(nbytes, buffer)
File "/usr/lib64/python3.7/ssl.py", line 929, in read
return self._sslobj.read(len, buffer)
ssl.SSLError: [SSL: UNKNOWN_STATE] unknown state (_ssl.c:2570)
File "/var/app/current/application/services/push.py", line 114, in send_message
response = messaging.send_multicast(message)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/firebase_admin/messaging.py", line 167, in send_multicast
return _get_messaging_service(app).send_all(messages, dry_run)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/firebase_admin/messaging.py", line 390, in send_all
batch.execute()
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/_helpers.py", line 134, in positional_wrapper
return wrapped(*args, **kwargs)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/http.py", line 1540, in execute
self._execute(http, self._order, self._requests)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/http.py", line 1470, in _execute
self._batch_uri, method="POST", body=body, headers=headers
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/google_auth_httplib2.py", line 201, in request
uri, method, body=body, headers=request_headers, **kwargs)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1994, in request
cachekey,
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1651, in _request
conn, request_uri, method, body, headers
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1618, in _conn_request
content = response.read()
File "/usr/lib64/python3.7/http/client.py", line 468, in read
return self._readall_chunked()
File "/usr/lib64/python3.7/http/client.py", line 575, in _readall_chunked
chunk_left = self._get_chunk_left()
File "/usr/lib64/python3.7/http/client.py", line 563, in _get_chunk_left
self._read_and_discard_trailer()
File "/usr/lib64/python3.7/http/client.py", line 536, in _read_and_discard_trailer
line = self.fp.readline(_MAXLINE + 1)
AttributeError: 'NoneType' object has no attribute 'readline'
Some other exceptions are as follows:
ssl.SSLError: [SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:2570)
ssl.SSLError: [SSL: DECRYPTION_FAILED_OR_BAD_RECORD_MAC] decryption failed or bad record mac (_ssl.c:2570)
ssl.SSLError: [SSL: BLOCK_CIPHER_PAD_IS_WRONG] block cipher pad is wrong (_ssl.c:2570)
The code looks like this:
message = messaging.MulticastMessage(
tokens=tokens,
data=data,
notification=messaging.Notification(
title=title,
body=body
)
)
response = messaging.send_multicast(message)
This is quite strange for me since the tasks are failing and as a result, retrying. But somehow I am receiving multiple notifications. That means the notifications are being sent but the tasks are getting failed. Please help me out in understanding what could be the problem. Does it have something to do with the system date time?
I am trying to deploy Rest Api with Chalice.
When I do chalice deploy.
The code worked with chalice local. But it fails in Deployment.
I get the following error:
Creating deployment package.
Updating policy for IAM role: learnChalice-dev
Updating lambda function: learnChalice-dev
Traceback (most recent call last):
File "/home/dragon/.local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/home/dragon/.local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 354, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/http/client.py", line 1252, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/awsrequest.py", line 92, in _send_request
method, url, body, headers, *args, **kwargs)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/http/client.py", line 1298, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/http/client.py", line 1247, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/awsrequest.py", line 119, in _send_output
self.send(msg)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/awsrequest.py", line 203, in send
return super(AWSConnection, self).send(str)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/http/client.py", line 987, in send
self.sock.sendall(data)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/ssl.py", line 1034, in sendall
v = self.send(byte_view[count:])
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/ssl.py", line 1003, in send
return self._sslobj.write(data)
socket.timeout: The write operation timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/httpsession.py", line 263, in send
chunked=self._chunked(request.headers),
File "/home/dragon/.local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 638, in urlopen
_stacktrace=sys.exc_info()[2])
File "/home/dragon/.local/lib/python3.7/site-packages/urllib3/util/retry.py", line 344, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/home/dragon/.local/lib/python3.7/site-packages/urllib3/packages/six.py", line 685, in reraise
raise value.with_traceback(tb)
File "/home/dragon/.local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/home/dragon/.local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 354, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/http/client.py", line 1252, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/awsrequest.py", line 92, in _send_request
method, url, body, headers, *args, **kwargs)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/http/client.py", line 1298, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/http/client.py", line 1247, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/awsrequest.py", line 119, in _send_output
self.send(msg)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/awsrequest.py", line 203, in send
return super(AWSConnection, self).send(str)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/http/client.py", line 987, in send
self.sock.sendall(data)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/ssl.py", line 1034, in sendall
v = self.send(byte_view[count:])
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/ssl.py", line 1003, in send
return self._sslobj.write(data)
urllib3.exceptions.ProtocolError: ('Connection aborted.', timeout('The write operation timed out'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/chalice/cli/__init__.py", line 599, in main
return cli(obj={})
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/click/core.py", line 829, in __call__
return self.main(*args, **kwargs)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/click/core.py", line 782, in main
rv = self.invoke(ctx)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/click/core.py", line 1259, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/click/core.py", line 1066, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/click/core.py", line 610, in invoke
return callback(*args, **kwargs)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/click/decorators.py", line 21, in new_func
return f(get_current_context(), *args, **kwargs)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/chalice/cli/init.py", line 206, in deploy
deployed_values = d.deploy(config, chalice_stage_name=stage)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/chalice/deploy/deployer.py", line 353, in deploy
return self._deploy(config, chalice_stage_name)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/chalice/deploy/deployer.py", line 366, in _deploy
self._executor.execute(plan)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/chalice/deploy/executor.py", line 44, in execute
self._default_handler)(instruction)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/chalice/deploy/executor.py", line 56, in _do_apicall
result = method(**final_kwargs)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/chalice/awsclient.py", line 273, in update_function
zip_contents=zip_contents)
File "/home/dragon/anaconda3/envs/my_env/lib/python3.7/site-packages/chalice/awsclient.py", line 294, in _update_function_code
FunctionName=function_name, ZipFile=zip_contents)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/client.py", line 316, in _api_call
return self._make_api_call(operation_name, kwargs)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/client.py", line 613, in _make_api_call
operation_model, request_dict, request_context)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/client.py", line 632, in _make_request
return self._endpoint.make_request(operation_model, request_dict)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/endpoint.py", line 102, in make_request
return self._send_request(request_dict, operation_model)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/endpoint.py", line 137, in _send_request
success_response, exception):
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/endpoint.py", line 231, in _needs_retry
caught_exception=caught_exception, request_dict=request_dict)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/hooks.py", line 356, in emit
return self._emitter.emit(aliased_event_name, **kwargs)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/hooks.py", line 228, in emit
return self._emit(event_name, kwargs)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/hooks.py", line 211, in _emit
response = handler(**kwargs)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/retryhandler.py", line 183, in __call__
if self._checker(attempts, response, caught_exception):
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/retryhandler.py", line 251, in __call__
caught_exception)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/retryhandler.py", line 277, in _should_retry
return self._checker(attempt_number, response, caught_exception)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/retryhandler.py", line 317, in __call__
caught_exception)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/retryhandler.py", line 223, in __call__
attempt_number, caught_exception)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/retryhandler.py", line 359, in _check_caught_exception
raise caught_exception
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/endpoint.py", line 200, in _do_get_response
http_response = self._send(request)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/endpoint.py", line 244, in _send
return self.http_session.send(request)
File "/home/dragon/.local/lib/python3.7/site-packages/botocore/httpsession.py", line 294, in send
endpoint_url=request.url
botocore.exceptions.ConnectionClosedError: Connection was closed before we received a valid response from endpoint URL: "https://lambda.ap-south-1.amazonaws.com/2015-03-31/functions/learnChalice-dev/code".
Thanks a lot for help.
Based on the comments, the solution was to increase default timeout:
chalice deploy --connection-timeout 300
I'm using Gmail API to send an email with attachments in Python 3.
I'm trying the same code as google developers as shown below:
https://developers.google.com/gmail/api/guides/sending
The problem is that when attachments are 4.2KB or 2.6MB, the code works Well; but when attachments are 3.0MB or 9.6MB or bigger, an Error occurs:
Traceback (most recent call last):
File "quickstart2.py", line 184, in <module>
main()
File "quickstart2.py", line 170, in main
send_message(service, "me", message)
File "quickstart2.py", line 147, in send_message
message = (service.users().messages().send(userId=user_id, body=message).execute())
File "/home/yizhu/anaconda3/lib/python3.6/site-packages/oauth2client/_helpers.py", line 133, in positional_wrapper
return wrapped(*args, **kwargs)
File "/home/yizhu/anaconda3/lib/python3.6/site-packages/googleapiclient/http.py", line 837, in execute
method=str(self.method), body=self.body, headers=self.headers)
File "/home/yizhu/anaconda3/lib/python3.6/site-packages/googleapiclient/http.py", line 176, in _retry_request
raise exception
File "/home/yizhu/anaconda3/lib/python3.6/site-packages/googleapiclient/http.py", line 163, in _retry_request
resp, content = http.request(uri, method, *args, **kwargs)
File "/home/yizhu/anaconda3/lib/python3.6/site-packages/oauth2client/transport.py", line 175, in new_request
redirections, connection_type)
File "/home/yizhu/anaconda3/lib/python3.6/site-packages/oauth2client/transport.py", line 282, in request
connection_type=connection_type)
File "/home/yizhu/anaconda3/lib/python3.6/site-packages/httplib2/__init__.py", line 1322, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "/home/yizhu/anaconda3/lib/python3.6/site-packages/httplib2/__init__.py", line 1072, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "/home/yizhu/anaconda3/lib/python3.6/site-packages/httplib2/__init__.py", line 996, in _conn_request
conn.request(method, request_uri, body, headers)
File "/home/yizhu/anaconda3/lib/python3.6/http/client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/home/yizhu/anaconda3/lib/python3.6/http/client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/home/yizhu/anaconda3/lib/python3.6/http/client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/home/yizhu/anaconda3/lib/python3.6/http/client.py", line 1065, in _send_output
self.send(chunk)
File "/home/yizhu/anaconda3/lib/python3.6/http/client.py", line 986, in send
self.sock.sendall(data)
File "/home/yizhu/anaconda3/lib/python3.6/ssl.py", line 972, in sendall
v = self.send(byte_view[count:])
File "/home/yizhu/anaconda3/lib/python3.6/ssl.py", line 941, in send
return self._sslobj.write(data)
File "/home/yizhu/anaconda3/lib/python3.6/ssl.py", line 642, in write
return self._sslobj.write(data)
BrokenPipeError: [Errno 32] Broken pipe
What's the problem here?
Thanks
It seems an exception is raised at _retry_request.
I haven't encountered this error myself, but there's a thread on github discussing about the same error.
https://github.com/google/google-api-python-client/issues/218
Try using httplib2shim, it seems oauth2client is still not replaced by google-auth.
Another suggestion I found was to use the MEDIA /upload option for files over 10 MB. Docs for how to use /upload: https://developers.google.com/gmail/api/v1/reference/users/messages/send
I am trying to set a user defined timeout on message pull with 'returnImmediately' = False :
PUBSUB_SCOPES = ['https://www.googleapis.com/auth/pubsub']
credentials = oauth2client.GoogleCredentials.get_application_default()
if credentials.create_scoped_required():
credentials = credentials.create_scoped(PUBSUB_SCOPES)
http = httplib2.Http(timeout=timeout)
credentials.authorize(http)
return discovery.build('pubsub', 'v1', http=http)
When the timeout is < 90 seconds I get the following errors:
resp = client.projects().subscriptions().pull(subscription=subscription, body=body).execute()
File "venv\lib\site-packages\oauth2client\util.py", line 137, in positional_wrapper
return wrapped(*args, **kwargs)
File "venv\lib\site-packages\googleapiclient\http.py", line 755, in execute
method=str(self.method), body=self.body, headers=self.headers)
File "venv\lib\site-packages\googleapiclient\http.py", line 93, in _retry_request
resp, content = http.request(uri, method, *args, **kwargs)
File "venv\lib\site-packages\oauth2client\client.py", line 622, in new_request
redirections, connection_type)
File "venv\lib\site-packages\httplib2\__init__.py", line 1609, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "venv\lib\site-packages\httplib2\__init__.py", line 1351, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "venv\lib\site-packages\httplib2\__init__.py", line 1307, in _conn_request
response = conn.getresponse()
File "C:\python27\Lib\httplib.py", line 1074, in getresponse
response.begin()
File "C:\python27\Lib\httplib.py", line 415, in begin
version, status, reason = self._read_status()
File "C:\python27\Lib\httplib.py", line 371, in _read_status
line = self.fp.readline(_MAXLINE + 1)
File "C:\python27\Lib\socket.py", line 476, in readline
data = self._sock.recv(self._rbufsize)
File "C:\python27\Lib\ssl.py", line 714, in recv
return self.read(buflen)
File "C:\python27\Lib\ssl.py", line 608, in read
v = self._sslobj.read(len or 1024)
SSLError: ('The read operation timed out',)
Thanks.
Unfortunately these client libraries do not support forwarding the timeout values to the server; however, we have just announced the gRPC client libraries, which correctly pass the deadline to the server.
As a workaround for the current libraries, either use returnImmediately=true, or set a deadline higher than 90 seconds, as your question implies.
I have a python 3.5 app that makes calls to Google Cloud Storage using the python sdk.
Every once in a while, for 10-30 minutes, all calls to the API fail with BrokenPipeError or ssl.SSLError errors. After some time, they just start working again, I have not noticed a pattern as to why.
Is this a known issue? Is it specific to the python sdk or is this a real performance regression on the side of google?
It should also be noted that these errors will emanate from the same code running on my local machine as well as from a GCE machine.
The trace for BrokenPipe:
Traceback (most recent call last):
File "oauth2client/util.py", line 140, in positional_wrapper
return wrapped(*args, **kwargs)
File "googleapiclient/http.py", line 722, in execute
body=self.body, headers=self.headers)
File "oauth2client/client.py", line 596, in new_request
redirections, connection_type)
File "httplib2/__init__.py", line 1314, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "httplib2/__init__.py", line 1064, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "httplib2/__init__.py", line 988, in _conn_request
conn.request(method, request_uri, body, headers)
File "/usr/lib/python3.5/http/client.py", line 1083, in request
self._send_request(method, url, body, headers)
File "/usr/lib/python3.5/http/client.py", line 1128, in _send_request
self.endheaders(body)
File "/usr/lib/python3.5/http/client.py", line 1079, in endheaders
self._send_output(message_body)
File "/usr/lib/python3.5/http/client.py", line 911, in _send_output
self.send(msg)
File "/usr/lib/python3.5/http/client.py", line 885, in send
self.sock.sendall(data)
File "/usr/lib/python3.5/ssl.py", line 886, in sendall
v = self.send(data[count:])
File "/usr/lib/python3.5/ssl.py", line 856, in send
return self._sslobj.write(data)
File "/usr/lib/python3.5/ssl.py", line 581, in write
return self._sslobj.write(data)
BrokenPipeError[Errno 32] Broken pipe
The trace for ssl.SSLError:
File "oauth2client/util.py", line 140, in positional_wrapper
return wrapped(*args, **kwargs)
File "googleapiclient/http.py", line 722, in execute
body=self.body, headers=self.headers)
File "oauth2client/client.py", line 596, in new_request
redirections, connection_type)
File "httplib2/__init__.py", line 1314, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "httplib2/__init__.py", line 1064, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "httplib2/__init__.py", line 1017, in _conn_request
response = conn.getresponse()
File "/usr/lib/python3.5/http/client.py", line 1174, in getresponse
response.begin()
File "/usr/lib/python3.5/http/client.py", line 282, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.5/http/client.py", line 243, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/lib/python3.5/socket.py", line 575, in readinto
return self._sock.recv_into(b)
File "/usr/lib/python3.5/ssl.py", line 924, in recv_into
return self.read(nbytes, buffer)
File "/usr/lib/python3.5/ssl.py", line 786, in read
return self._sslobj.read(len, buffer)
File "/usr/lib/python3.5/ssl.py", line 570, in read
v = self._sslobj.read(len, buffer)
ssl.SSLError[SSL: DECRYPTION_FAILED_OR_BAD_RECORD_MAC] decryption failed or bad record mac (_ssl.c:1974)
Definitely looks like an intermittent issue on Googles side.
The broken pipe issue relates to httpclient2 being unable to reconnect an existing connection to their API, this is the error that has the greatest impact on our services. We also on few occasions received "503 Backend Error".
Our "solution" was to basically allow the connections to close themselves by releasing the client once done and creating a new one for the next request.
Bare in mind though that our requests are very sparse, services using Cloud Storage as primary storage probably wants to keep the connections open for as long as possible.