I'm trying to use pygsheets in a script on ScrapingHub. The pygsheets part of the script begins with:
google_client = pygsheets.authorize(service_file=CREDENTIALS_FILENAME, no_cache=True)
spreadsheet = google_client.open_by_key(SHEET_ID)
Where CREDENTIALS_FILENAME is my Google service account credentials file name and SHEET_ID is the spreadsheet ID; at some point of that, this is happening:
2018-01-30 16:27:36 INFO [stdout] service_email : *******#**********.iam.gserviceaccount.com
2018-01-30 16:27:36 INFO [googleapiclient.discovery] URL being requested: GET https://www.googleapis.com/drive/v3/files?corpora=user&pageSize=500&fields=files%28id%2C+name%29&q=mimeType%3D%27application%2Fvnd.google-apps.spreadsheet%27&supportsTeamDrives=false&includeTeamDriveItems=false&alt=json
2018-01-30 16:27:36 INFO [oauth2client.transport] Attempting refresh to obtain initial access_token
2018-01-30 16:27:36 INFO [oauth2client.client] Refreshing access_token
2018-01-30 16:31:50 ERROR [root] Job runtime exception
Traceback (most recent call last):
File "/app/python/lib/python3.6/site-packages/httplib2/__init__.py", line 995, in _conn_request
conn.connect()
File "/usr/local/lib/python3.6/http/client.py", line 1392, in connect
super().connect()
File "/usr/local/lib/python3.6/http/client.py", line 936, in connect
(self.host,self.port), self.timeout, self.source_address)
File "/usr/local/lib/python3.6/socket.py", line 704, in create_connection
for res in getaddrinfo(host, port, 0, SOCK_STREAM):
File "/usr/local/lib/python3.6/socket.py", line 745, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno -2] Name or service not known
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/sh_scrapy/crawl.py", line 148, in _run_usercode
_run(args, settings)
File "/usr/local/lib/python3.6/site-packages/sh_scrapy/crawl.py", line 105, in _run
_run_pkgscript(args)
File "/usr/local/lib/python3.6/site-packages/sh_scrapy/crawl.py", line 128, in _run_pkgscript
d.run_script(scriptname, {'__name__': '__main__'})
File "/usr/local/lib/python3.6/site-packages/pkg_resources/__init__.py", line 1517, in run_script
exec(code, namespace, namespace)
File "/tmp/unpacked-eggs/__main__.egg/EGG-INFO/scripts/update_gsheets.py", line 190, in <module>
main(parse_args())
File "/tmp/unpacked-eggs/__main__.egg/EGG-INFO/scripts/update_gsheets.py", line 152, in main
google_client = pygsheets.authorize(service_file=CREDENTIALS_FILENAME, no_cache=True)
File "/app/python/lib/python3.6/site-packages/pygsheets/client.py", line 552, in authorize
rclient = Client(oauth=credentials, **client_kwargs)
File "/app/python/lib/python3.6/site-packages/pygsheets/client.py", line 79, in __init__
self._fetch_sheets()
File "/app/python/lib/python3.6/site-packages/pygsheets/client.py", line 92, in _fetch_sheets
results = self._execute_request(None, request, False)
File "/app/python/lib/python3.6/site-packages/pygsheets/client.py", line 418, in _execute_request
response = request.execute()
File "/app/python/lib/python3.6/site-packages/oauth2client/_helpers.py", line 133, in positional_wrapper
return wrapped(*args, **kwargs)
File "/app/python/lib/python3.6/site-packages/googleapiclient/http.py", line 839, in execute
method=str(self.method), body=self.body, headers=self.headers)
File "/app/python/lib/python3.6/site-packages/googleapiclient/http.py", line 166, in _retry_request
resp, content = http.request(uri, method, *args, **kwargs)
File "/app/python/lib/python3.6/site-packages/oauth2client/transport.py", line 175, in new_request
redirections, connection_type)
File "/app/python/lib/python3.6/site-packages/oauth2client/transport.py", line 282, in request
connection_type=connection_type)
File "/app/python/lib/python3.6/site-packages/httplib2/__init__.py", line 1322, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "/app/python/lib/python3.6/site-packages/httplib2/__init__.py", line 1072, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "/app/python/lib/python3.6/site-packages/httplib2/__init__.py", line 1002, in _conn_request
raise ServerNotFoundError("Unable to find the server at %s" % conn.host)
httplib2.ServerNotFoundError: Unable to find the server at www.googleapis.com
I've tested the script before and it used to work, so I don't know what is happening. What is this happening? There is some configuration or authentication step that I missed?
Related
Just to give a brief introduction, I am using elastic beanstalk from Amazon to deploy my python (Falcon) application. I have two environments of elastic beanstalk (one being the main app server and the other one a worker environment). The worker environment receives HTTP requests for async tasks and then executes them. Most of the tasks are either push notifications (mobile devices) or sending emails. The strange part is that my tasks are failing with errors that I am unable to determine the cause for (this works fine for my local environment). Some of the tracebacks look like this:
self.send_message(message=message)
File "/var/app/current/application/services/push.py", line 114, in send_message
response = messaging.send_multicast(message)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/firebase_admin/messaging.py", line 167, in send_multicast
return _get_messaging_service(app).send_all(messages, dry_run)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/firebase_admin/messaging.py", line 390, in send_all
batch.execute()
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/_helpers.py", line 134, in positional_wrapper
return wrapped(*args, **kwargs)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/http.py", line 1540, in execute
self._execute(http, self._order, self._requests)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/http.py", line 1470, in _execute
self._batch_uri, method="POST", body=body, headers=headers
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/google_auth_httplib2.py", line 201, in request
uri, method, body=body, headers=request_headers, **kwargs)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1994, in request
cachekey,
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1651, in _request
conn, request_uri, method, body, headers
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1589, in _conn_request
response = conn.getresponse()
File "/usr/lib64/python3.7/http/client.py", line 1369, in getresponse
response.begin()
File "/usr/lib64/python3.7/http/client.py", line 310, in begin
version, status, reason = self._read_status()
File "/usr/lib64/python3.7/http/client.py", line 271, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/lib64/python3.7/socket.py", line 589, in readinto
return self._sock.recv_into(b)
File "/usr/lib64/python3.7/ssl.py", line 1071, in recv_into
return self.read(nbytes, buffer)
File "/usr/lib64/python3.7/ssl.py", line 929, in read
return self._sslobj.read(len, buffer)
socket.timeout: The read operation timed out
File "/var/app/current/application/services/push.py", line 152, in execute_push
self.send_message(message=message)
File "/var/app/current/application/services/push.py", line 114, in send_message
response = messaging.send_multicast(message)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/firebase_admin/messaging.py", line 167, in send_multicast
return _get_messaging_service(app).send_all(messages, dry_run)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/firebase_admin/messaging.py", line 390, in send_all
batch.execute()
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/_helpers.py", line 134, in positional_wrapper
return wrapped(*args, **kwargs)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/http.py", line 1540, in execute
self._execute(http, self._order, self._requests)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/http.py", line 1470, in _execute
self._batch_uri, method="POST", body=body, headers=headers
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/google_auth_httplib2.py", line 201, in request
uri, method, body=body, headers=request_headers, **kwargs)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1994, in request
cachekey,
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1651, in _request
conn, request_uri, method, body, headers
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1589, in _conn_request
response = conn.getresponse()
File "/usr/lib64/python3.7/http/client.py", line 1369, in getresponse
response.begin()
File "/usr/lib64/python3.7/http/client.py", line 310, in begin
version, status, reason = self._read_status()
File "/usr/lib64/python3.7/http/client.py", line 271, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/lib64/python3.7/socket.py", line 589, in readinto
return self._sock.recv_into(b)
File "/usr/lib64/python3.7/ssl.py", line 1071, in recv_into
return self.read(nbytes, buffer)
File "/usr/lib64/python3.7/ssl.py", line 929, in read
return self._sslobj.read(len, buffer)
ssl.SSLError: [SSL: UNKNOWN_STATE] unknown state (_ssl.c:2570)
File "/var/app/current/application/services/push.py", line 114, in send_message
response = messaging.send_multicast(message)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/firebase_admin/messaging.py", line 167, in send_multicast
return _get_messaging_service(app).send_all(messages, dry_run)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/firebase_admin/messaging.py", line 390, in send_all
batch.execute()
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/_helpers.py", line 134, in positional_wrapper
return wrapped(*args, **kwargs)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/http.py", line 1540, in execute
self._execute(http, self._order, self._requests)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/googleapiclient/http.py", line 1470, in _execute
self._batch_uri, method="POST", body=body, headers=headers
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/google_auth_httplib2.py", line 201, in request
uri, method, body=body, headers=request_headers, **kwargs)
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1994, in request
cachekey,
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1651, in _request
conn, request_uri, method, body, headers
File "/var/app/venv/staging-LQM1lest/lib/python3.7/site-packages/httplib2/__init__.py", line 1618, in _conn_request
content = response.read()
File "/usr/lib64/python3.7/http/client.py", line 468, in read
return self._readall_chunked()
File "/usr/lib64/python3.7/http/client.py", line 575, in _readall_chunked
chunk_left = self._get_chunk_left()
File "/usr/lib64/python3.7/http/client.py", line 563, in _get_chunk_left
self._read_and_discard_trailer()
File "/usr/lib64/python3.7/http/client.py", line 536, in _read_and_discard_trailer
line = self.fp.readline(_MAXLINE + 1)
AttributeError: 'NoneType' object has no attribute 'readline'
Some other exceptions are as follows:
ssl.SSLError: [SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:2570)
ssl.SSLError: [SSL: DECRYPTION_FAILED_OR_BAD_RECORD_MAC] decryption failed or bad record mac (_ssl.c:2570)
ssl.SSLError: [SSL: BLOCK_CIPHER_PAD_IS_WRONG] block cipher pad is wrong (_ssl.c:2570)
The code looks like this:
message = messaging.MulticastMessage(
tokens=tokens,
data=data,
notification=messaging.Notification(
title=title,
body=body
)
)
response = messaging.send_multicast(message)
This is quite strange for me since the tasks are failing and as a result, retrying. But somehow I am receiving multiple notifications. That means the notifications are being sent but the tasks are getting failed. Please help me out in understanding what could be the problem. Does it have something to do with the system date time?
today I'm try to use yahoo-finance library than some issue occur by code.
i provide code and occurred error.
here is library link https://pypi.org/project/yahoo-finance/
code:-
from yahoo_finance import Share
yahoo = Share('YHOO')
print(yahoo.get_open())
Error is
Traceback (most recent call last):
File "/usr/lib/python3.6/urllib/request.py", line 1318, in do_open
encode_chunked=req.has_header('Transfer-encoding'))
File "/usr/lib/python3.6/http/client.py", line 1254, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1300, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1249, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1036, in _send_output
self.send(msg)
File "/usr/lib/python3.6/http/client.py", line 974, in send
self.connect()
File "/usr/lib/python3.6/http/client.py", line 1407, in connect
super().connect()
File "/usr/lib/python3.6/http/client.py", line 946, in connect
(self.host,self.port), self.timeout, self.source_address)
File "/usr/lib/python3.6/socket.py", line 704, in create_connection
for res in getaddrinfo(host, port, 0, SOCK_STREAM):
File "/usr/lib/python3.6/socket.py", line 745, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno -2] Name or service not known
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "main.py", line 3, in <module>
yahoo = Share('YHOO')
File "/home/yogesh/.local/lib/python3.6/site-packages/yahoo_finance/__init__.py", line 178, in __init__
self.refresh()
File "/home/yogesh/.local/lib/python3.6/site-packages/yahoo_finance/__init__.py", line 142, in refresh
self.data_set = self._fetch()
File "/home/yogesh/.local/lib/python3.6/site-packages/yahoo_finance/__init__.py", line 181, in _fetch
data = super(Share, self)._fetch()
File "/home/yogesh/.local/lib/python3.6/site-packages/yahoo_finance/__init__.py", line 134, in _fetch
data = self._request(query)
File "/home/yogesh/.local/lib/python3.6/site-packages/yahoo_finance/__init__.py", line 118, in _request
response = yql.YQLQuery().execute(query)
File "/home/yogesh/.local/lib/python3.6/site-packages/yahoo_finance/yql.py", line 61, in execute
'env': DATATABLES_URL
File "/usr/lib/python3.6/urllib/request.py", line 223, in urlopen
return opener.open(url, data, timeout)
File "/usr/lib/python3.6/urllib/request.py", line 526, in open
response = self._open(req, data)
File "/usr/lib/python3.6/urllib/request.py", line 544, in _open
'_open', req)
File "/usr/lib/python3.6/urllib/request.py", line 504, in _call_chain
result = func(*args)
File "/usr/lib/python3.6/urllib/request.py", line 1361, in https_open
context=self._context, check_hostname=self._check_hostname)
File "/usr/lib/python3.6/urllib/request.py", line 1320, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error [Errno -2] Name or service not known>
i have a no idea about this error, please help me to solve this problem. Thank you....
The package is broken. See for example, this issue on GitHub: https://github.com/lukaszbanasiak/yahoo-finance/issues/148
Your best bet would be to indicate on GitHub that the package is not working and perhaps the author is able to help you further.
I'm trying to install an application developed by someone else and when running a manage.py command I'm getting this error, which I do not understand. I wonder if anyone can tell me where to look to investigate first - thank you.
POST http://elasticsearch:9200/_bulk?refresh=true [status:N/A request:0.001s]
Traceback (most recent call last):
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/django/db/models/query.py", line 486, in get_or_create
return self.get(**lookup), False
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/djmoney/models/managers.py", line 209, in wrapper
queryset = func(*args, **kwargs)
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/django/db/models/query.py", line 399, in get
self.model._meta.object_name
providers.models.Organisation.DoesNotExist: Organisation matching query does not exist.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/connection.py", line 160, in _new_conn
(self._dns_host, self.port), self.timeout, **extra_kw)
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/util/connection.py", line 57, in create_connection
for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
File "/usr/lib/python3.6/socket.py", line 745, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno -2] Name or service not known
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/elasticsearch/connection/http_urllib3.py", line 95, in perform_request
response = self.pool.urlopen(method, url, body, retries=False, headers=self.headers, **kw)
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/connectionpool.py", line 641, in urlopen
_stacktrace=sys.exc_info()[2])
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/util/retry.py", line 344, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/packages/six.py", line 686, in reraise
raise value
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/connectionpool.py", line 603, in urlopen
chunked=chunked)
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/connectionpool.py", line 355, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/lib/python3.6/http/client.py", line 1254, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1300, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1249, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1036, in _send_output
self.send(msg)
File "/usr/lib/python3.6/http/client.py", line 974, in send
self.connect()
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/connection.py", line 183, in connect
conn = self._new_conn()
File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/connection.py", line 169, in _new_conn
self, "Failed to establish a new connection: %s" % e)
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x7f506c988eb8>: Failed to establish a new connection: [Errno -2] Name or service not known
EDIT
Added manage.py as requested
#!/usr/bin/env python
import os
import sys
if __name__ == "__main__":
os.environ.setdefault('DEPLOY_ENV', 'local')
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings.%s' % os.environ['DEPLOY_ENV'])
from django.core.management import execute_from_command_line # noqa
execute_from_command_line(sys.argv)
As Goran said, I was trying to connect to elasticsearch:9200 but testing localhost:9200
We have a kubernetes cron job on GCP that submits several copies of the same Python dataflow job, each in its own container. Whenever we need a new copy of the job, we just add it to the spec->jobTemplate->spec->template->spec->containers part of the cron job yaml and adjust the dataflow job parameters as needed. This usually works fine, but the latest copy we tried to add does not work. The existing copies are all still working as expected. The job seems to fail on submission to GCP, and the error message is not very helpful:
Traceback (most recent call last):
File "/app/job.py", line 117, in <module>
newness.pipeline.run_dataflow(sys.argv)
File "/app/newness/pipeline.py", line 480, in run_dataflow
result = pipe.run()
File "/usr/local/lib/python2.7/dist-packages/apache_beam/pipeline.py", line 403, in run
self.to_runner_api(), self.runner, self._options).run(False)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/pipeline.py", line 416, in run
return self.runner.run_pipeline(self)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/dataflow/dataflow_runner.py", line 389, in run_pipeline
self.dataflow_client.create_job(self.job), self)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/utils/retry.py", line 184, in wrapper
return fun(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/dataflow/internal/apiclient.py", line 504, in create_job
return self.submit_job_description(job)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/utils/retry.py", line 184, in wrapper
return fun(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/dataflow/internal/apiclient.py", line 551, in submit_job_description
response = self._client.projects_locations_jobs.Create(request)
File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py", line 578, in Create
config, request, global_params=global_params)
File "/usr/local/lib/python2.7/dist-packages/apitools/base/py/base_api.py", line 731, in _RunMethod
return self.ProcessHttpResponse(method_config, http_response, request)
File "/usr/local/lib/python2.7/dist-packages/apitools/base/py/base_api.py", line 737, in ProcessHttpResponse
self.__ProcessHttpResponse(method_config, http_response, request))
File "/usr/local/lib/python2.7/dist-packages/apitools/base/py/base_api.py", line 604, in __ProcessHttpResponse
http_response, method_config=method_config, request=request)
apitools.base.py.exceptions.HttpError: <exception str() failed>
The job does not appear in the dataflow console at all.
The previous lines of the container logs look like:
2019-10-13T03:57:47.725542287Z Successfully downloaded apache-beam
2019-10-13T03:58:17.125601280Z INFO:root:Staging SDK sources from PyPI to gs://gcs-bucket-name/staging/newness-boosting-c2898.1570936519.827087/dataflow_python_sdk.tar
2019-10-13T03:58:17.324843623Z INFO:root:Starting GCS upload to gs://gcs-bucket-name/staging/newness-boosting-c2898.1570936519.827087/dataflow_python_sdk.tar...
2019-10-13T03:58:24.825657227Z INFO:root:Completed GCS upload to gs://gcs-bucket-name/staging/newness-boosting-c2898.1570936519.827087/dataflow_python_sdk.tar
2019-10-13T03:58:25.225646529Z INFO:root:Downloading binary distribtution of the SDK from PyPi
2019-10-13T03:58:25.225716554Z INFO:root:Executing command: ['/usr/bin/python', '-m', 'pip', 'download', '--dest', '/tmp/tmpk5TfMS', 'apache-beam==2.8.0', '--no-deps', '--only-binary', ':all:', '--python-version', '27', '--implementation', 'cp', '--abi', 'cp27mu', '--platform', 'manylinux1_x86_64']
2019-10-13T03:59:33.926186906Z Collecting apache-beam==2.8.0
2019-10-13T03:59:52.125678183Z Using cached https://files.pythonhosted.org/packages/0f/63/ea5453ba656d060936acf41d2ec057f23aafd69649e2129ac66fdda67d48/apache_beam-2.8.0-cp27-cp27mu-manylinux1_x86_64.whl
2019-10-13T04:00:11.525435891Z Saved /tmp/tmpk5TfMS/apache_beam-2.8.0-cp27-cp27mu-manylinux1_x86_64.whl
2019-10-13T04:00:12.025054706Z Successfully downloaded apache-beam
2019-10-13T04:00:26.726190542Z INFO:root:Staging binary distribution of the SDK from PyPI to gs://gcs-bucket-name/staging/newness-boosting-c2898.1570936519.827087/apache_beam-2.8.0-cp27-cp27mu-manylinux1_x86_64.whl
2019-10-13T04:00:26.825618945Z INFO:root:Starting GCS upload to gs://gcs-bucket-name/staging/newness-boosting-c2898.1570936519.827087/apache_beam-2.8.0-cp27-cp27mu-manylinux1_x86_64.whl...
2019-10-13T04:00:33.725522899Z INFO:root:Completed GCS upload to gs://gcs-bucket-name/staging/newness-boosting-c2898.1570936519.827087/apache_beam-2.8.0-cp27-cp27mu-manylinux1_x86_64.whl
2019-10-13T04:06:14.525017097Z Traceback (most recent call last):
...
Why is this job failing to submit? Are there any other logs we can look at to see the cause of this failure?
(Most of our dataflow jobs are written in Java, where the error messages are usually more helpful.)
UPDATE: Running job locally (Windows) with apache-beam 2.16 has the same issue but more logging detail:
...
INFO:root:Starting GCS upload to gs://gcs-bucket-name/staging/newness-boosting-c2898.1571606418.971000/apache_beam-2.16.0-cp27-cp27mu-manylinux1_x86_64.whl...
INFO:root:Completed GCS upload to gs://gcs-bucket-name/staging/newness-boosting-c2898.1571606418.971000/apache_beam-2.16.0-cp27-cp27mu-manylinux1_x86_64.whl in 3 seconds.
WARNING:root:Discarding unparseable args: ['job.py', '--days_history=30']
WARNING:root:Discarding unparseable args: ['job.py', '--days_history=30']
WARNING:root:Retry with exponential backoff: waiting for 2.64795143823 seconds before retrying submit_job_description because we caught exception: error: [Errno 10053] An established connection was aborted by the software in your host machine
Traceback for above exception (most recent call last):
File "C:\Python27\lib\site-packages\apache_beam\utils\retry.py", line 206, in wrapper
return fun(*args, **kwargs)
File "C:\Python27\lib\site-packages\apache_beam\runners\dataflow\internal\apiclient.py", line 593, in submit_job_description
response = self._client.projects_locations_jobs.Create(request)
File "C:\Python27\lib\site-packages\apache_beam\runners\dataflow\internal\clients\dataflow\dataflow_v1b3_client.py", line 657, in Create
config, request, global_params=global_params)
File "C:\Python27\lib\site-packages\apitools\base\py\base_api.py", line 729, in _RunMethod
http, http_request, **opts)
File "C:\Python27\lib\site-packages\apitools\base\py\http_wrapper.py", line 346, in MakeRequest
check_response_func=check_response_func)
File "C:\Python27\lib\site-packages\apitools\base\py\http_wrapper.py", line 396, in _MakeRequestNoRetry
redirections=redirections, connection_type=connection_type)
File "C:\Python27\lib\site-packages\oauth2client\transport.py", line 169, in new_request
redirections, connection_type)
File "C:\Python27\lib\site-packages\oauth2client\transport.py", line 169, in new_request
redirections, connection_type)
File "C:\Python27\lib\site-packages\httplib2\__init__.py", line 1694, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "C:\Python27\lib\site-packages\httplib2\__init__.py", line 1434, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "C:\Python27\lib\site-packages\httplib2\__init__.py", line 1390, in _conn_request
response = conn.getresponse()
File "C:\Python27\lib\httplib.py", line 1121, in getresponse
response.begin()
File "C:\Python27\lib\httplib.py", line 438, in begin
version, status, reason = self._read_status()
File "C:\Python27\lib\httplib.py", line 394, in _read_status
line = self.fp.readline(_MAXLINE + 1)
File "C:\Python27\lib\socket.py", line 480, in readline
data = self._sock.recv(self._rbufsize)
File "C:\Python27\lib\ssl.py", line 754, in recv
return self.read(buflen)
File "C:\Python27\lib\ssl.py", line 641, in read
v = self._sslobj.read(len)
... retries 4 times total ...
Traceback (most recent call last):
File "job.py", line 117, in <module>
newness.pipeline.run_dataflow(sys.argv)
File "C:\Users\LeeW\Desktop\newness\newness\pipeline.py", line 480, in run_dataflow
result = pipe.run()
File "C:\Python27\lib\site-packages\apache_beam\pipeline.py", line 407, in run
self._options).run(False)
File "C:\Python27\lib\site-packages\apache_beam\pipeline.py", line 420, in run
return self.runner.run_pipeline(self, self._options)
File "C:\Python27\lib\site-packages\apache_beam\runners\dataflow\dataflow_runner.py", line 485, in run_pipeline
self.dataflow_client.create_job(self.job), self)
File "C:\Python27\lib\site-packages\apache_beam\utils\retry.py", line 206, in wrapper
return fun(*args, **kwargs)
File "C:\Python27\lib\site-packages\apache_beam\runners\dataflow\internal\apiclient.py", line 546, in create_job
return self.submit_job_description(job)
File "C:\Python27\lib\site-packages\apache_beam\utils\retry.py", line 219, in wrapper
raise_with_traceback(exn, exn_traceback)
File "C:\Python27\lib\site-packages\apache_beam\utils\retry.py", line 206, in wrapper
return fun(*args, **kwargs)
File "C:\Python27\lib\site-packages\apache_beam\runners\dataflow\internal\apiclient.py", line 593, in submit_job_description
response = self._client.projects_locations_jobs.Create(request)
File "C:\Python27\lib\site-packages\apache_beam\runners\dataflow\internal\clients\dataflow\dataflow_v1b3_client.py", line 657, in Create
config, request, global_params=global_params)
File "C:\Python27\lib\site-packages\apitools\base\py\base_api.py", line 729, in _RunMethod
http, http_request, **opts)
File "C:\Python27\lib\site-packages\apitools\base\py\http_wrapper.py", line 346, in MakeRequest
check_response_func=check_response_func)
File "C:\Python27\lib\site-packages\apitools\base\py\http_wrapper.py", line 396, in _MakeRequestNoRetry
redirections=redirections, connection_type=connection_type)
File "C:\Python27\lib\site-packages\oauth2client\transport.py", line 169, in new_request
redirections, connection_type)
File "C:\Python27\lib\site-packages\oauth2client\transport.py", line 169, in new_request
redirections, connection_type)
File "C:\Python27\lib\site-packages\httplib2\__init__.py", line 1694, in request
(response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey)
File "C:\Python27\lib\site-packages\httplib2\__init__.py", line 1434, in _request
(response, content) = self._conn_request(conn, request_uri, method, body, headers)
File "C:\Python27\lib\site-packages\httplib2\__init__.py", line 1390, in _conn_request
response = conn.getresponse()
File "C:\Python27\lib\httplib.py", line 1121, in getresponse
response.begin()
File "C:\Python27\lib\httplib.py", line 438, in begin
version, status, reason = self._read_status()
File "C:\Python27\lib\httplib.py", line 394, in _read_status
line = self.fp.readline(_MAXLINE + 1)
File "C:\Python27\lib\socket.py", line 480, in readline
data = self._sock.recv(self._rbufsize)
File "C:\Python27\lib\ssl.py", line 754, in recv
return self.read(buflen)
File "C:\Python27\lib\ssl.py", line 641, in read
v = self._sslobj.read(len)
socket.error: [Errno 10053] An established connection was aborted by the software in your host machine
Which version of Beam Python SDK are you using?
I copy pasted a code from another post which for them, seems fine but mine prints an error and I don't know why... How to make urllib2 requests through Tor in Python?
Anyone can help me with this error?
import socks
import socket
socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, "127.0.0.1", 9150)
socket.socket = socks.socksocket
import urllib.request
print(urllib.request.urlopen('http://www.google.com').read())
Traceback (most recent call last):
File "/Users/Raphael/Desktop/whatsmyip.py", line 17, in <module>
r = requests.get("http://checkip.dyn.com")
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/requests/api.py", line 68, in get
return request('get', url, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/requests/api.py", line 50, in request
response = session.request(method=method, url=url, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/requests/sessions.py", line 465, in request
resp = self.send(prep, **send_kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/requests/sessions.py", line 573, in send
r = adapter.send(request, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/requests/adapters.py", line 370, in send
timeout=timeout
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/requests/packages/urllib3/connectionpool.py", line 544, in urlopen
body=body, headers=headers)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/requests/packages/urllib3/connectionpool.py", line 349, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/http/client.py", line 1088, in request
self._send_request(method, url, body, headers)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/http/client.py", line 1126, in _send_request
self.endheaders(body)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/http/client.py", line 1084, in endheaders
self._send_output(message_body)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/http/client.py", line 922, in _send_output
self.send(msg)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/http/client.py", line 857, in send
self.connect()
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/requests/packages/urllib3/connection.py", line 155, in connect
conn = self._new_conn()
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/requests/packages/urllib3/connection.py", line 134, in _new_conn
(self.host, self.port), self.timeout, **extra_kw)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/requests/packages/urllib3/util/connection.py", line 78, in create_connection
sock.connect(sa)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/socks.py", line 369, in connect
self.__negotiatesocks5(destpair[0],destpair[1])
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/socks.py", line 176, in __negotiatesocks5
self.sendall("\x05\x01\x00")
TypeError: 'str' does not support the buffer interface
and do anyone know why? I would really like to use this module but it doesn't work!