python mrjob - gaierror: [Errno -2] Name or service not known - python

I'm trying to access s3 files from the mrjob module.
Here's the code that's failing:
from mrjob.emr import S3Filesystem
fs = S3Filesystem("<aws_access_key_id>", "<aws_secret_access_key>", "us-west-2")
fs.get_s3_key("s3n://<bucket>/<folder>/file.txt")
(but with the actual aws_access_key_id, aws_secret_access_key, bucket, and folder) and the error I'm getting is:
Traceback (most recent call last):
File "<input>", line 1, in <module>
File "/usr/local/lib/python2.7/dist-packages/mrjob/fs/s3.py", line 235, in get_s3_key
bucket = s3_conn.get_bucket(bucket_name)
File "/usr/local/lib/python2.7/dist-packages/mrjob/retry.py", line 80, in call_and_maybe_retry
return f(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/boto/s3/connection.py", line 431, in get_bucket
bucket.get_all_keys(headers, maxkeys=0)
File "/usr/local/lib/python2.7/dist-packages/boto/s3/bucket.py", line 375, in get_all_keys
'', headers, **params)
File "/usr/local/lib/python2.7/dist-packages/boto/s3/bucket.py", line 332, in _get_all
query_args=query_args)
File "/usr/local/lib/python2.7/dist-packages/boto/s3/connection.py", line 547, in make_request
retry_handler=retry_handler
File "/usr/local/lib/python2.7/dist-packages/boto/connection.py", line 947, in make_request
retry_handler=retry_handler)
File "/usr/local/lib/python2.7/dist-packages/boto/connection.py", line 908, in _mexe
raise e
gaierror: [Errno -2] Name or service not known
Any suggestions?
Thanks!


This (s3n://)
fs.get_s3_key("s3n://<bucket>/<folder>/file.txt")
should be this (s3://) :
fs.get_s3_key("s3://<bucket>/<folder>/file.txt")

Related

Module 'cryptography.utils' has no attribute 'bit_length'

I am currently trying to figure out how to use netmiko to automate some of my routine work.Such as getting configuration backup, creating vlans etc.I've managed to use it on Aruba and Huawei Switches without problem.But on alcatel switch I'm facing this issue:
Unknown exception: module 'cryptography.utils' has no attribute 'bit_length'
Traceback (most recent call last):
File "C:\Users\melih.celik\AppData\Local\Programs\Python\Python39\lib\site-packages\paramiko\transport.py", line 2075, in run
self.kex_engine.parse_next(ptype, m)
File "C:\Users\melih.celik\AppData\Local\Programs\Python\Python39\lib\site-packages\paramiko\kex_gex.py", line 101, in parse_next
return self._parse_kexdh_gex_reply(m)
File "C:\Users\melih.celik\AppData\Local\Programs\Python\Python39\lib\site-packages\paramiko\kex_gex.py", line 281, in _parse_kexdh_gex_reply
self.transport._verify_key(host_key, sig)
File "C:\Users\melih.celik\AppData\Local\Programs\Python\Python39\lib\site-packages\paramiko\transport.py", line 1886, in _verify_key
if not key.verify_ssh_sig(self.H, Message(sig)):
File "C:\Users\melih.celik\AppData\Local\Programs\Python\Python39\lib\site-packages\paramiko\dsskey.py", line 148, in verify_ssh_sig
key = dsa.DSAPublicNumbers(
File "C:\Users\melih.celik\AppData\Local\Programs\Python\Python39\lib\site-packages\cryptography\hazmat\primitives\asymmetric\dsa.py", line 212, in public_key
return backend.load_dsa_public_numbers(self)
File "C:\Users\melih.celik\AppData\Local\Programs\Python\Python39\lib\site-packages\cryptography\hazmat\backends\openssl\backend.py", line 871, in load_dsa_public_numbers
dsa._check_dsa_parameters(numbers.parameter_numbers)
File "C:\Users\melih.celik\AppData\Local\Programs\Python\Python39\lib\site-packages\netmiko\tplink\tplink_jetstream.py", line 145, in _override_check_dsa_parameters
if crypto_utils.bit_length(parameters.q) not in [160, 256]:
AttributeError: module 'cryptography.utils' has no attribute 'bit_length'
Traceback (most recent call last):
File "C:\Users\melih.celik\Desktop\New_Backup\Yedek\Coding\Rand stuff\ssh_deneme(toplu).py", line 75, in
config_backup(cihaz_secim,ip_address,username,password)
File "C:\Users\melih.celik\Desktop\New_Backup\Yedek\Coding\Rand stuff\ssh_deneme(toplu).py", line 12, in config_backup
net_connect=ConnectHandler(**switch) #Baglanti kuruldu.
File "C:\Users\melih.celik\AppData\Local\Programs\Python\Python39\lib\site-packages\netmiko\ssh_dispatcher.py", line 312, in ConnectHandler
return ConnectionClass(*args, **kwargs)
File "C:\Users\melih.celik\AppData\Local\Programs\Python\Python39\lib\site-packages\netmiko\base_connection.py", line 346, in init
self._open()
File "C:\Users\melih.celik\AppData\Local\Programs\Python\Python39\lib\site-packages\netmiko\base_connection.py", line 351, in _open
self.establish_connection()
File "C:\Users\melih.celik\AppData\Local\Programs\Python\Python39\lib\site-packages\netmiko\base_connection.py", line 920, in establish_connection
self.remote_conn_pre.connect(**ssh_connect_params)
File "C:\Users\melih.celik\AppData\Local\Programs\Python\Python39\lib\site-packages\paramiko\client.py", line 406, in connect
t.start_client(timeout=timeout)
File "C:\Users\melih.celik\AppData\Local\Programs\Python\Python39\lib\site-packages\paramiko\transport.py", line 660, in start_client
raise e
File "C:\Users\melih.celik\AppData\Local\Programs\Python\Python39\lib\site-packages\paramiko\transport.py", line 2075, in run
self.kex_engine.parse_next(ptype, m)
File "C:\Users\melih.celik\AppData\Local\Programs\Python\Python39\lib\site-packages\paramiko\kex_gex.py", line 101, in parse_next
return self._parse_kexdh_gex_reply(m)
File "C:\Users\melih.celik\AppData\Local\Programs\Python\Python39\lib\site-packages\paramiko\kex_gex.py", line 281, in _parse_kexdh_gex_reply
self.transport._verify_key(host_key, sig)
File "C:\Users\melih.celik\AppData\Local\Programs\Python\Python39\lib\site-packages\paramiko\transport.py", line 1886, in _verify_key
if not key.verify_ssh_sig(self.H, Message(sig)):
File "C:\Users\melih.celik\AppData\Local\Programs\Python\Python39\lib\site-packages\paramiko\dsskey.py", line 148, in verify_ssh_sig
key = dsa.DSAPublicNumbers(
File "C:\Users\melih.celik\AppData\Local\Programs\Python\Python39\lib\site-packages\cryptography\hazmat\primitives\asymmetric\dsa.py", line 212, in public_key
return backend.load_dsa_public_numbers(self)
File "C:\Users\melih.celik\AppData\Local\Programs\Python\Python39\lib\site-packages\cryptography\hazmat\backends\openssl\backend.py", line 871, in load_dsa_public_numbers
dsa._check_dsa_parameters(numbers.parameter_numbers)
File "C:\Users\melih.celik\AppData\Local\Programs\Python\Python39\lib\site-packages\netmiko\tplink\tplink_jetstream.py", line 145, in _override_check_dsa_parameters
if crypto_utils.bit_length(parameters.q) not in [160, 256]:
AttributeError: module 'cryptography.utils' has no attribute 'bit_length'
Thanks for your help in advance.

It looks like tplink_jetstream.py assumes that a (now removed) helper function is available to it. The simplest fix here would be to go into that file and modify the line containing crypto_utils.bit_length(parameters.q) to read parameters.q.bit_length() instead.

PyCharm's code coverage not working due to Windows drive letters

I have a computer that has a C and D drive, where PyCharm, Python and the source code are installed on the D drive. I'm using Pipenv with PIPENV_VENV_IN_PROJECT set to enabled so that also ends up in the D drive. Despite all this, when running with code coverage enabled, I get this error:
Destroying test database for alias 'default'...
Traceback (most recent call last):
File "D:\Development\PyCharm\PyCharm 2019.1.1\helpers\coverage_runner\run_coverage.py", line 54, in <module>
main()
File "D:\Business\projectx\.venv\lib\site-packages\coverage\cmdline.py", line 770, in main
status = CoverageScript().command_line(argv)
File "D:\Business\projectx\.venv\lib\site-packages\coverage\cmdline.py", line 489, in command_line
return self.do_run(options, args)
File "D:\Business\projectx\.venv\lib\site-packages\coverage\cmdline.py", line 657, in do_run
self.coverage.save()
File "D:\Business\projectx\.venv\lib\site-packages\coverage\control.py", line 529, in save
data = self.get_data()
File "D:\Business\projectx\.venv\lib\site-packages\coverage\control.py", line 583, in get_data
if self._collector and self._collector.flush_data():
File "D:\Business\projectx\.venv\lib\site-packages\coverage\collector.py", line 425, in flush_data
self.covdata.add_lines(abs_file_dict(self.data))
File "D:\Business\projectx\.venv\lib\site-packages\coverage\sqldata.py", line 236, in add_lines
self._choose_lines_or_arcs(lines=True)
File "D:\Business\projectx\.venv\lib\site-packages\coverage\sqldata.py", line 279, in _choose_lines_or_arcs
with self._connect() as con:
File "D:\Business\projectx\.venv\lib\site-packages\coverage\sqldata.py", line 160, in _connect
self._create_db()
File "D:\Business\projectx\.venv\lib\site-packages\coverage\sqldata.py", line 117, in _create_db
with self._db:
File "D:\Business\projectx\.venv\lib\site-packages\coverage\sqldata.py", line 525, in __enter__
self.connect()
File "D:\Business\projectx\.venv\lib\site-packages\coverage\sqldata.py", line 510, in connect
filename = os.path.relpath(self.filename)
File "D:\Business\projectx\.venv\lib\ntpath.py", line 562, in relpath
path_drive, start_drive))
ValueError: path is on mount 'C:', start on mount 'D:'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "D:\Development\PyCharm\PyCharm 2019.1.1\helpers\coverage_runner\run_coverage.py", line 58, in <module>
main(["xml", "-o", coverage_file + ".xml", "--ignore-errors"])
File "D:\Business\projectx\.venv\lib\site-packages\coverage\cmdline.py", line 770, in main
status = CoverageScript().command_line(argv)
File "D:\Business\projectx\.venv\lib\site-packages\coverage\cmdline.py", line 511, in command_line
self.coverage.load()
File "D:\Business\projectx\.venv\lib\site-packages\coverage\control.py", line 336, in load
self._data.read()
File "D:\Business\projectx\.venv\lib\site-packages\coverage\sqldata.py", line 409, in read
with self._connect(): # TODO: doesn't look right
File "D:\Business\projectx\.venv\lib\site-packages\coverage\sqldata.py", line 160, in _connect
self._create_db()
File "D:\Business\projectx\.venv\lib\site-packages\coverage\sqldata.py", line 117, in _create_db
with self._db:
File "D:\Business\projectx\.venv\lib\site-packages\coverage\sqldata.py", line 525, in __enter__
self.connect()
File "D:\Business\projectx\.venv\lib\site-packages\coverage\sqldata.py", line 510, in connect
filename = os.path.relpath(self.filename)
File "D:\Business\projectx\.venv\lib\ntpath.py", line 562, in relpath
path_drive, start_drive))
ValueError: path is on mount 'C:', start on mount 'D:'
Any ideas why?

Mongodb crashes when the same collection in accessed by two different python script using pymongo

I have two python scripts, currentdataupload.py and productioncount.py on my Ubuntu server. currentdataupload.py uploads dummy current data and productioncount.py accesses that data to do some calculations. The code runs perfectly in other scenarios without disrupting the MongoDB, but when I run both of these codes simultaneously using nohup after a few seconds it exits the mongoDB service and is no longer able to upload or retrieve data. I would like to know if I am doing the entire process the right way or I am messing up somewhere big time.
Error log
Traceback (most recent call last):
File "/python_codes/currentdataupload.py", line 38, in <module>
result = posts.insert_one(post_data)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/collection.py", line 630, in insert_one
bypass_doc_val=bypass_document_validation),
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/collection.py", line 535, in _insert
check_keys, manipulate, write_concern, op_id, bypass_doc_val)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/collection.py", line 516, in _insert_one
check_keys=check_keys)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 244, in command
self._raise_connection_failure(error)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 372, in _raise_connection_failure
raise error
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 239, in command
read_concern)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/network.py", line 96, in command
response = receive_message(sock, 1, request_id)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/network.py", line 123, in receive_message
header = _receive_data_on_socket(sock, 16)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/network.py", line 161, in _receive_data_on_socket
raise AutoReconnect("connection closed")
pymongo.errors.AutoReconnect: connection closed
Traceback (most recent call last):
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 543, in connect
sock = _configured_socket(self.address, self.opts)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 452, in _configured_socket
sock = _create_connection(address, options)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 436, in _create_connection
raise err
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 429, in _create_connection
sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/python_codes/productioncount.py", line 267, in <module>
result = col2.find_one({"date": date, "machine": machine})
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/collection.py", line 1014, in find_one
for result in cursor.limit(-1):
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/cursor.py", line 1090, in next
if len(self.__data) or self._refresh():
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/cursor.py", line 1012, in _refresh
self.__read_concern))
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/cursor.py", line 850, in __send_message
**kwargs)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/mongo_client.py", line 844, in _send_message_with_response
exhaust)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/mongo_client.py", line 855, in _reset_on_error
return func(*args, **kwargs)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/server.py", line 99, in send_message_with_response
with self.get_socket(all_credentials, exhaust) as sock_info:
File "/root/anaconda3/lib/python3.5/contextlib.py", line 59, in __enter__
return next(self.gen)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/server.py", line 163, in get_socket
with self.pool.get_socket(all_credentials, checkout) as sock_info:
File "/root/anaconda3/lib/python3.5/contextlib.py", line 59, in __enter__
return next(self.gen)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 582, in get_socket
sock_info = self._get_socket_no_auth()
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 628, in _get_socket_no_auth
sock_info = self._check(sock_info)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 682, in _check
return self.connect()
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 555, in connect
_raise_connection_failure(self.address, error)
File "/root/anaconda3/lib/python3.5/site-packages/pymongo/pool.py", line 65, in _raise_connection_failure
raise AutoReconnect(msg)
pymongo.errors.AutoReconnect: localhost:27017: [Errno 111] Connection refused

MongoDB shouldn't crash. Please file a bug report, including the Python script you run and a complete MongoDB logfile, at jira.mongodb.org.

Celery throwing long error message

I'm trying to run the starter code for celery on their website (http://docs.celeryproject.org/en/latest/getting-started/first-steps-with-celery.html), and I'm running an instance of a RabbitMQ server in the background. However, I'm getting a long error message:
>>> from celery import Celery
>>> app = Celery('tasks', broker='pyamqp://guest#localhost//')
>>> #app.task
... def add(x, y):
... return x + y
...
>>>
>>> add.delay(4, 4)
Traceback (most recent call last):
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\utils\functional.py", line 36, in __call__
return self.__value__
AttributeError: 'ChannelPromise' object has no attribute '__value__'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\connection.py", line 494, in _ensured
return fun(*args, **kwargs)
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\messaging.py", line 187, in _publish
channel = self.channel
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\messaging.py", line 209, in _get_channel
channel = self._channel = channel()
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\utils\functional.py", line 38, in __call__
value = self.__value__ = self.__contract__()
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\messaging.py", line 224, in <lambda>
channel = ChannelPromise(lambda: connection.default_channel)
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\connection.py", line 819, in default_channel
self.connection
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\connection.py", line 802, in connection
self._connection = self._establish_connection()
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\connection.py", line 757, in _establish_connection
conn = self.transport.establish_connection()
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\transport\pyamqp.py", line 130, in establish_connection
conn.connect()
File "C:\Users\Alexander\Anaconda3\lib\site-packages\amqp\connection.py", line 294, in connect
self.transport.connect()
File "C:\Users\Alexander\Anaconda3\lib\site-packages\amqp\transport.py", line 122, in connect
self.socket_settings, self.read_timeout, self.write_timeout,
File "C:\Users\Alexander\Anaconda3\lib\site-packages\amqp\transport.py", line 174, in _init_socket
self._set_socket_options(socket_settings)
File "C:\Users\Alexander\Anaconda3\lib\site-packages\amqp\transport.py", line 204, in _set_socket_options
self.sock.setsockopt(SOL_TCP, opt, val)
OSError: [WinError 10042] An unknown, invalid, or unsupported option or level was specified in a getsockopt or setsockopt call
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\connection.py", line 414, in _reraise_as_library_errors
yield
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\connection.py", line 515, in _ensured
reraise_as_library_errors=False,
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\connection.py", line 405, in ensure_connection
callback)
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\utils\functional.py", line 333, in retry_over_time
return fun(*args, **kwargs)
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\connection.py", line 261, in connect
return self.connection
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\connection.py", line 802, in connection
self._connection = self._establish_connection()
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\connection.py", line 757, in _establish_connection
conn = self.transport.establish_connection()
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\transport\pyamqp.py", line 130, in establish_connection
conn.connect()
File "C:\Users\Alexander\Anaconda3\lib\site-packages\amqp\connection.py", line 294, in connect
self.transport.connect()
File "C:\Users\Alexander\Anaconda3\lib\site-packages\amqp\transport.py", line 122, in connect
self.socket_settings, self.read_timeout, self.write_timeout,
File "C:\Users\Alexander\Anaconda3\lib\site-packages\amqp\transport.py", line 174, in _init_socket
self._set_socket_options(socket_settings)
File "C:\Users\Alexander\Anaconda3\lib\site-packages\amqp\transport.py", line 204, in _set_socket_options
self.sock.setsockopt(SOL_TCP, opt, val)
OSError: [WinError 10042] An unknown, invalid, or unsupported option or level was specified in a getsockopt or setsockopt call
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "C:\Users\Alexander\Anaconda3\lib\site-packages\celery\app\task.py", line 412, in delay
return self.apply_async(args, kwargs)
File "C:\Users\Alexander\Anaconda3\lib\site-packages\celery\app\task.py", line 535, in apply_async
**options
File "C:\Users\Alexander\Anaconda3\lib\site-packages\celery\app\base.py", line 737, in send_task
amqp.send_task_message(P, name, message, **options)
File "C:\Users\Alexander\Anaconda3\lib\site-packages\celery\app\amqp.py", line 558, in send_task_message
**properties
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\messaging.py", line 181, in publish
exchange_name, declare,
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\connection.py", line 527, in _ensured
errback and errback(exc, 0)
File "C:\Users\Alexander\Anaconda3\lib\contextlib.py", line 77, in __exit__
self.gen.throw(type, value, traceback)
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\connection.py", line 419, in _reraise_as_library_errors
sys.exc_info()[2])
File "C:\Users\Alexander\Anaconda3\lib\site-packages\vine\five.py", line 175, in reraise
raise value.with_traceback(tb)
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\connection.py", line 414, in _reraise_as_library_errors
yield
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\connection.py", line 515, in _ensured
reraise_as_library_errors=False,
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\connection.py", line 405, in ensure_connection
callback)
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\utils\functional.py", line 333, in retry_over_time
return fun(*args, **kwargs)
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\connection.py", line 261, in connect
return self.connection
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\connection.py", line 802, in connection
self._connection = self._establish_connection()
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\connection.py", line 757, in _establish_connection
conn = self.transport.establish_connection()
File "C:\Users\Alexander\Anaconda3\lib\site-packages\kombu\transport\pyamqp.py", line 130, in establish_connection
conn.connect()
File "C:\Users\Alexander\Anaconda3\lib\site-packages\amqp\connection.py", line 294, in connect
self.transport.connect()
File "C:\Users\Alexander\Anaconda3\lib\site-packages\amqp\transport.py", line 122, in connect
self.socket_settings, self.read_timeout, self.write_timeout,
File "C:\Users\Alexander\Anaconda3\lib\site-packages\amqp\transport.py", line 174, in _init_socket
self._set_socket_options(socket_settings)
File "C:\Users\Alexander\Anaconda3\lib\site-packages\amqp\transport.py", line 204, in _set_socket_options
self.sock.setsockopt(SOL_TCP, opt, val)
kombu.exceptions.OperationalError: [WinError 10042] An unknown, invalid, or unsupported option or level was specified in a getsockopt or setsockopt call

There is an open issue on github where the same OS-error has been seen:
https://github.com/celery/py-amqp/issues/130
Someone suggested in the comments on the issue that it is temporarily resolved by downgrading the version of amqp to 2.1.3
Steps to take in order to downgrade:
Uninstall amqp using $ pip uninstall amqp.
Install amqp using $ pip install -Iv amqp==2.1.3.

socket.gaierror when trying to run emr using python mrjob

I currently trying to learn mrjob and how to implement it in AWS EMR so please forgive me if I am asking already asked question [searched many places but did not find the answer] and sorry if it is a silly question
This is my python script :
from mrjob.job import MRJob
class MRWordFrequencyCount(MRJob):
def mapper(self, _, line):
yield "chars", len(line)
yield "words", len(line.split())
yield "lines", 1
def reducer(self, key, values):
yield key, sum(values)
if __name__ == '__main__':
MRWordFrequencyCount.run()
When I run it in local mode I got the result
cmd:
python sample.py input.txt
So I tried to run this in EMR
by creating a mrjob.conf file
which looks like this :
runners:
emr:
aws_access_key_id:
aws_secret_access_key:
aws_region: us-west-2a
ec2_key_pair: emr
ec2_key_pair_file: ~/Desktop/emr.pem
ec2_instance_type: m1.small
num_ec2_instances: 5
local:
base_tmp_dir: /tmp
First attempt
Trying it locally on my windows system
python check.py -r emr --conf-path ./mrjob.conf word.txt
Note :
Same error came when I kept the input in s3 location and gave it as an argument
I got this traceback:
Traceback (most recent call last):
File "check.py", line 16, in <module>
MRWordFrequencyCount.run()
File "C:\Users\MOB140003207\AppData\Local\Enthought\Canopy32\User\lib\site-pac
kages\mrjob\job.py", line 461, in run
mr_job.execute()
File "C:\Users\MOB140003207\AppData\Local\Enthought\Canopy32\User\lib\site-pac
kages\mrjob\job.py", line 479, in execute
super(MRJob, self).execute()
File "C:\Users\MOB140003207\AppData\Local\Enthought\Canopy32\User\lib\site-pac
kages\mrjob\launch.py", line 153, in execute
self.run_job()
File "C:\Users\MOB140003207\AppData\Local\Enthought\Canopy32\User\lib\site-pac
kages\mrjob\launch.py", line 215, in run_job
with self.make_runner() as runner:
File "C:\Users\MOB140003207\AppData\Local\Enthought\Canopy32\User\lib\site-pac
kages\mrjob\job.py", line 502, in make_runner
return super(MRJob, self).make_runner()
File "C:\Users\MOB140003207\AppData\Local\Enthought\Canopy32\User\lib\site-pac
kages\mrjob\launch.py", line 168, in make_runner
return EMRJobRunner(**self.emr_job_runner_kwargs())
File "C:\Users\MOB140003207\AppData\Local\Enthought\Canopy32\User\lib\site-pac
kages\mrjob\emr.py", line 643, in __init__
self._fix_s3_scratch_and_log_uri_opts()
File "C:\Users\MOB140003207\AppData\Local\Enthought\Canopy32\User\lib\site-pac
kages\mrjob\emr.py", line 760, in _fix_s3_scratch_and_log_uri_opts
self._set_s3_scratch_uri(s3_conn)
File "C:\Users\MOB140003207\AppData\Local\Enthought\Canopy32\User\lib\site-pac
kages\mrjob\emr.py", line 776, in _set_s3_scratch_uri
buckets = s3_conn.get_all_buckets()
File "C:\Users\MOB140003207\AppData\Local\Enthought\Canopy32\User\lib\site-pac
kages\mrjob\retry.py", line 149, in call_and_maybe_retry
return f(*args, **kwargs)
File "C:\Users\MOB140003207\AppData\Local\Enthought\Canopy32\User\lib\site-pac
kages\boto\s3\connection.py", line 436, in get_all_buckets
response = self.make_request('GET', headers=headers)
File "C:\Users\MOB140003207\AppData\Local\Enthought\Canopy32\User\lib\site-pac
kages\boto\s3\connection.py", line 664, in make_request
retry_handler=retry_handler
File "C:\Users\MOB140003207\AppData\Local\Enthought\Canopy32\User\lib\site-pac
kages\boto\connection.py", line 1070, in make_request
retry_handler=retry_handler)
File "C:\Users\MOB140003207\AppData\Local\Enthought\Canopy32\User\lib\site-pac
kages\boto\connection.py", line 1029, in _mexe
raise ex
socket.gaierror: [Errno 11004] getaddrinfo failed
When I tried to run it in aws EC2 instance
I got this error
Traceback (most recent call last):
File "check.py", line 16, in <module>
MRWordFrequencyCount.run()
File "/usr/local/lib/python2.7/dist-packages/mrjob/job.py", line 461, in run
mr_job.execute()
File "/usr/local/lib/python2.7/dist-packages/mrjob/job.py", line 479, in execute
super(MRJob, self).execute()
File "/usr/local/lib/python2.7/dist-packages/mrjob/launch.py", line 153, in execute
self.run_job()
File "/usr/local/lib/python2.7/dist-packages/mrjob/launch.py", line 215, in run_job
with self.make_runner() as runner:
File "/usr/local/lib/python2.7/dist-packages/mrjob/job.py", line 502, in make_runner
return super(MRJob, self).make_runner()
File "/usr/local/lib/python2.7/dist-packages/mrjob/launch.py", line 168, in make_runner
return EMRJobRunner(**self.emr_job_runner_kwargs())
File "/usr/local/lib/python2.7/dist-packages/mrjob/emr.py", line 643, in __init__
self._fix_s3_scratch_and_log_uri_opts()
File "/usr/local/lib/python2.7/dist-packages/mrjob/emr.py", line 760, in _fix_s3_scratch_and_log_uri_opts
self._set_s3_scratch_uri(s3_conn)
File "/usr/local/lib/python2.7/dist-packages/mrjob/emr.py", line 776, in _set_s3_scratch_uri
buckets = s3_conn.get_all_buckets()
File "/usr/local/lib/python2.7/dist-packages/mrjob/retry.py", line 149, in call_and_maybe_retry
return f(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/boto/s3/connection.py", line 436, in get_all_buckets
response = self.make_request('GET', headers=headers)
File "/usr/local/lib/python2.7/dist-packages/boto/s3/connection.py", line 664, in make_request
retry_handler=retry_handler
File "/usr/local/lib/python2.7/dist-packages/boto/connection.py", line 1071, in make_request
retry_handler=retry_handler)
File "/usr/local/lib/python2.7/dist-packages/boto/connection.py", line 1030, in _mexe
raise ex
socket.gaierror: [Errno -2] Name or service not known
I don't know what I am doing wrong
python version 2.7 mrjob version '0.4.5'

After hours of searching and investigating I found the problem
It was in this line
aws_region: us-west-2a
Where as it should have been
aws_region: us-west-2
I just want to keep the question alive since it may save other people time

Categories

Resources