Related
today I'm try to use yahoo-finance library than some issue occur by code.
i provide code and occurred error.
here is library link https://pypi.org/project/yahoo-finance/
code:-
from yahoo_finance import Share
yahoo = Share('YHOO')
print(yahoo.get_open())
Error is
Traceback (most recent call last):
File "/usr/lib/python3.6/urllib/request.py", line 1318, in do_open
encode_chunked=req.has_header('Transfer-encoding'))
File "/usr/lib/python3.6/http/client.py", line 1254, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1300, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1249, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1036, in _send_output
self.send(msg)
File "/usr/lib/python3.6/http/client.py", line 974, in send
self.connect()
File "/usr/lib/python3.6/http/client.py", line 1407, in connect
super().connect()
File "/usr/lib/python3.6/http/client.py", line 946, in connect
(self.host,self.port), self.timeout, self.source_address)
File "/usr/lib/python3.6/socket.py", line 704, in create_connection
for res in getaddrinfo(host, port, 0, SOCK_STREAM):
File "/usr/lib/python3.6/socket.py", line 745, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno -2] Name or service not known
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "main.py", line 3, in <module>
yahoo = Share('YHOO')
File "/home/yogesh/.local/lib/python3.6/site-packages/yahoo_finance/__init__.py", line 178, in __init__
self.refresh()
File "/home/yogesh/.local/lib/python3.6/site-packages/yahoo_finance/__init__.py", line 142, in refresh
self.data_set = self._fetch()
File "/home/yogesh/.local/lib/python3.6/site-packages/yahoo_finance/__init__.py", line 181, in _fetch
data = super(Share, self)._fetch()
File "/home/yogesh/.local/lib/python3.6/site-packages/yahoo_finance/__init__.py", line 134, in _fetch
data = self._request(query)
File "/home/yogesh/.local/lib/python3.6/site-packages/yahoo_finance/__init__.py", line 118, in _request
response = yql.YQLQuery().execute(query)
File "/home/yogesh/.local/lib/python3.6/site-packages/yahoo_finance/yql.py", line 61, in execute
'env': DATATABLES_URL
File "/usr/lib/python3.6/urllib/request.py", line 223, in urlopen
return opener.open(url, data, timeout)
File "/usr/lib/python3.6/urllib/request.py", line 526, in open
response = self._open(req, data)
File "/usr/lib/python3.6/urllib/request.py", line 544, in _open
'_open', req)
File "/usr/lib/python3.6/urllib/request.py", line 504, in _call_chain
result = func(*args)
File "/usr/lib/python3.6/urllib/request.py", line 1361, in https_open
context=self._context, check_hostname=self._check_hostname)
File "/usr/lib/python3.6/urllib/request.py", line 1320, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error [Errno -2] Name or service not known>
i have a no idea about this error, please help me to solve this problem. Thank you....
The package is broken. See for example, this issue on GitHub: https://github.com/lukaszbanasiak/yahoo-finance/issues/148
Your best bet would be to indicate on GitHub that the package is not working and perhaps the author is able to help you further.
I have this expectation of Elasticsearch.ping: it should return True if Elasticsearch can be contacted, and False if it cannot. However, if connection cannot be established, ping will display a bunch of errors before finally returning False. Is there a way to prevent it? I really don't need all that, I just want my False.
$ python3
Python 3.6.5 (default, Jun 17 2018, 12:13:06)
[GCC 4.2.1 Compatible Apple LLVM 9.1.0 (clang-902.0.39.2)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> import elasticsearch
>>> elasticsearch.Elasticsearch().ping()
HEAD http://localhost:9200/ [status:N/A request:0.002s]
Traceback (most recent call last):
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/connection.py", line 171, in _new_conn
(self._dns_host, self.port), self.timeout, **extra_kw)
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/util/connection.py", line 79, in create_connection
raise err
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/util/connection.py", line 69, in create_connection
sock.connect(sa)
ConnectionRefusedError: [Errno 61] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/elasticsearch/connection/http_urllib3.py", line 172, in perform_request
response = self.pool.urlopen(method, url, body, retries=Retry(False), headers=request_headers, **kw)
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/connectionpool.py", line 638, in urlopen
_stacktrace=sys.exc_info()[2])
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/util/retry.py", line 343, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/packages/six.py", line 686, in reraise
raise value
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/connectionpool.py", line 354, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1026, in _send_output
self.send(msg)
File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 964, in send
self.connect()
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/connection.py", line 196, in connect
conn = self._new_conn()
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/connection.py", line 180, in _new_conn
self, "Failed to establish a new connection: %s" % e)
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x109d19550>: Failed to establish a new connection: [Errno 61] Connection refused
HEAD http://localhost:9200/ [status:N/A request:0.002s]
Traceback (most recent call last):
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/connection.py", line 171, in _new_conn
(self._dns_host, self.port), self.timeout, **extra_kw)
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/util/connection.py", line 79, in create_connection
raise err
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/util/connection.py", line 69, in create_connection
sock.connect(sa)
ConnectionRefusedError: [Errno 61] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/elasticsearch/connection/http_urllib3.py", line 172, in perform_request
response = self.pool.urlopen(method, url, body, retries=Retry(False), headers=request_headers, **kw)
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/connectionpool.py", line 638, in urlopen
_stacktrace=sys.exc_info()[2])
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/util/retry.py", line 343, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/packages/six.py", line 686, in reraise
raise value
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/connectionpool.py", line 354, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1026, in _send_output
self.send(msg)
File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 964, in send
self.connect()
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/connection.py", line 196, in connect
conn = self._new_conn()
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/connection.py", line 180, in _new_conn
self, "Failed to establish a new connection: %s" % e)
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x109d195c0>: Failed to establish a new connection: [Errno 61] Connection refused
HEAD http://localhost:9200/ [status:N/A request:0.002s]
Traceback (most recent call last):
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/connection.py", line 171, in _new_conn
(self._dns_host, self.port), self.timeout, **extra_kw)
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/util/connection.py", line 79, in create_connection
raise err
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/util/connection.py", line 69, in create_connection
sock.connect(sa)
ConnectionRefusedError: [Errno 61] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/elasticsearch/connection/http_urllib3.py", line 172, in perform_request
response = self.pool.urlopen(method, url, body, retries=Retry(False), headers=request_headers, **kw)
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/connectionpool.py", line 638, in urlopen
_stacktrace=sys.exc_info()[2])
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/util/retry.py", line 343, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/packages/six.py", line 686, in reraise
raise value
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/connectionpool.py", line 354, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1026, in _send_output
self.send(msg)
File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 964, in send
self.connect()
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/connection.py", line 196, in connect
conn = self._new_conn()
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/connection.py", line 180, in _new_conn
self, "Failed to establish a new connection: %s" % e)
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x109d196a0>: Failed to establish a new connection: [Errno 61] Connection refused
HEAD http://localhost:9200/ [status:N/A request:0.002s]
Traceback (most recent call last):
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/connection.py", line 171, in _new_conn
(self._dns_host, self.port), self.timeout, **extra_kw)
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/util/connection.py", line 79, in create_connection
raise err
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/util/connection.py", line 69, in create_connection
sock.connect(sa)
ConnectionRefusedError: [Errno 61] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/elasticsearch/connection/http_urllib3.py", line 172, in perform_request
response = self.pool.urlopen(method, url, body, retries=Retry(False), headers=request_headers, **kw)
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/connectionpool.py", line 638, in urlopen
_stacktrace=sys.exc_info()[2])
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/util/retry.py", line 343, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/packages/six.py", line 686, in reraise
raise value
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/connectionpool.py", line 354, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1026, in _send_output
self.send(msg)
File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 964, in send
self.connect()
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/connection.py", line 196, in connect
conn = self._new_conn()
File "/Users/amadan/work/aistqa/venv/lib/python3.6/site-packages/urllib3/connection.py", line 180, in _new_conn
self, "Failed to establish a new connection: %s" % e)
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x109d19780>: Failed to establish a new connection: [Errno 61] Connection refused
False
>>>
Seems like an error that some people encountered :
https://github.com/elastic/elasticsearch-py/issues/666
to sum up the above discussion try and suppress the logger in elastic search to only log errors , like so:
import logging
logging.basicConfig(level=logging.ERROR)
and see if it helps you .
EDIT: the OP fixed the problem i a more concrete way to elasticsearch with :
import logging;
logging.getLogger('elasticsearch').level = logging.ERROR
both ways works.
For me the provided solution didn't work, since the starktrace is generated by urllib3. So for me this worked out:
import logging
logging.getLogger("urllib3").setLevel(logging.ERROR)
Hello everyone I am pretty new to programming so please forgive me any noob mistakes.
I am trying to use urllib in python 3.6 to find a string on a webpage and the solution I found here is working great for most webpages i tested, for example:
from urllib.request import urlopen
>>>string = 'google'
>>>url = 'https://google.com/'
>>>webp=urlopen(url).read()
>>>isfound = webp.find(string.encode('utf-8'))
>>>print(isfound)
186
But when I try using the URL https://chan.sankakucomplex.com/ (NSFW) I get tons of errors and I don't know why.
I allready tried changing my DNS to 8.8.8.8 but this didn't change anything.
error:
Traceback (most recent call last):
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 1318, in do_open
encode_chunked=req.has_header('Transfer-encoding'))
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 1026, in _send_output
self.send(msg)
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 964, in send
self.connect()
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 1392, in connect
super().connect()
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\http\client.py", line 936, in connect
(self.host,self.port), self.timeout, self.source_address)
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\socket.py", line 704, in create_connection
for res in getaddrinfo(host, port, 0, SOCK_STREAM):
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\socket.py", line 743, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno 11004] getaddrinfo failed
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:/Users/-/Desktop/python/searcher/stackex.py", line 5, in <module>
webp=urlopen(url).read()
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 223, in urlopen
return opener.open(url, data, timeout)
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 526, in open
response = self._open(req, data)
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 544, in _open
'_open', req)
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 504, in _call_chain
result = func(*args)
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 1361, in https_open
context=self._context, check_hostname=self._check_hostname)
File "C:\Users\-\AppData\Local\Programs\Python\Python36\lib\urllib\request.py", line 1320, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error [Errno 11004] getaddrinfo failed>
urllib.error.URLError: <urlopen error [Errno 11004] getaddrinfo failed>
This tells me that your DNS server is refusing to translate the text address (chan.sankakucomplex.com) into an IP address. It's not an issue with Python.
Can you get to that site from a web browser running on the same machine where you're running your python code?
I have created a single node HDFS in a VM (hadoop.master, IP: 192.168.12.52). The file etc/hadoop/core-site.xml has the following configuration for the namenode:
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://master.hadoop:9000/</value>
</property>
</configuration>
I want to read a file from the HDFS on my local, physical desktop. For that, this is my code, which I've saved in a file named hdfs_read.py:
from hdfs import InsecureClient
client = InsecureClient('http://192.168.12.52:9000')
with client.read('/opt/hadoop/LICENSE.txt') as reader:
features = reader.read()
print(features)
Now when I run it, I get the following timeout error:
$ python3 hdfs_read.py
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 137, in _new_conn
(self.host, self.port), self.timeout, **extra_kw)
File "/usr/lib/python3/dist-packages/urllib3/util/connection.py", line 91, in create_connection
raise err
File "/usr/lib/python3/dist-packages/urllib3/util/connection.py", line 81, in create_connection
sock.connect(sa)
OSError: [Errno 113] No route to host
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 560, in urlopen
body=body, headers=headers)
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 354, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/lib/python3.6/http/client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1026, in _send_output
self.send(msg)
File "/usr/lib/python3.6/http/client.py", line 964, in send
self.connect()
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 162, in connect
conn = self._new_conn()
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 146, in _new_conn
self, "Failed to establish a new connection: %s" % e)
requests.packages.urllib3.exceptions.NewConnectionError: <requests.packages.urllib3.connection.HTTPConnection object at 0x7f2d88cef2b0>: Failed to establish a new connection: [Errno 113] No route to host
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 376, in send
timeout=timeout
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 610, in urlopen
_stacktrace=sys.exc_info()[2])
File "/usr/lib/python3/dist-packages/urllib3/util/retry.py", line 273, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
requests.packages.urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='192.168.12.52', port=9000): Max retries exceeded with url: /webhdfs/v1/home/edhuser/testdata.txt?user.name=embs&offset=0&op=OPEN (Caused by NewConnectionError('<requests.packages.urllib3.connection.HTTPConnection object at 0x7f2d88cef2b0>: Failed to establish a new connection: [Errno 113] No route to host',))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "hdfs_read_local.py", line 3, in <module>
with client.read('/home/edhuser/testdata.txt') as reader:
File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
return next(self.gen)
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 678, in read
buffersize=buffer_size,
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 118, in api_handler
raise err
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 107, in api_handler
**self.kwargs
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 207, in _request
**kwargs
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 468, in request
resp = self.send(prep, **send_kwargs)
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 576, in send
r = adapter.send(request, **kwargs)
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 437, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='192.168.12.52', port=9000): Max retries exceeded with url: /webhdfs/v1/home/edhuser/testdata.txt?user.name=embs&offset=0&op=OPEN (Caused by NewConnectionError('<requests.packages.urllib3.connection.HTTPConnection object at 0x7f2d88cef2b0>: Failed to establish a new connection: [Errno 113] No route to host',))
Error in sys.excepthook:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/apport_python_hook.py", line 63, in apport_excepthook
from apport.fileutils import likely_packaged, get_recent_crashes
File "/usr/lib/python3/dist-packages/apport/__init__.py", line 5, in <module>
from apport.report import Report
File "/usr/lib/python3/dist-packages/apport/report.py", line 30, in <module>
import apport.fileutils
File "/usr/lib/python3/dist-packages/apport/fileutils.py", line 23, in <module>
from apport.packaging_impl import impl as packaging
File "/usr/lib/python3/dist-packages/apport/packaging_impl.py", line 23, in <module>
import apt
File "/usr/lib/python3/dist-packages/apt/__init__.py", line 23, in <module>
import apt_pkg
ModuleNotFoundError: No module named 'apt_pkg'
Original exception was:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 137, in _new_conn
(self.host, self.port), self.timeout, **extra_kw)
File "/usr/lib/python3/dist-packages/urllib3/util/connection.py", line 91, in create_connection
raise err
File "/usr/lib/python3/dist-packages/urllib3/util/connection.py", line 81, in create_connection
sock.connect(sa)
OSError: [Errno 113] No route to host
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 560, in urlopen
body=body, headers=headers)
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 354, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/lib/python3.6/http/client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1026, in _send_output
self.send(msg)
File "/usr/lib/python3.6/http/client.py", line 964, in send
self.connect()
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 162, in connect
conn = self._new_conn()
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 146, in _new_conn
self, "Failed to establish a new connection: %s" % e)
requests.packages.urllib3.exceptions.NewConnectionError: <requests.packages.urllib3.connection.HTTPConnection object at 0x7f2d88cef2b0>: Failed to establish a new connection: [Errno 113] No route to host
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 376, in send
timeout=timeout
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 610, in urlopen
_stacktrace=sys.exc_info()[2])
File "/usr/lib/python3/dist-packages/urllib3/util/retry.py", line 273, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
requests.packages.urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='192.168.12.52', port=9000): Max retries exceeded with url: /webhdfs/v1/home/edhuser/testdata.txt?user.name=embs&offset=0&op=OPEN (Caused by NewConnectionError('<requests.packages.urllib3.connection.HTTPConnection object at 0x7f2d88cef2b0>: Failed to establish a new connection: [Errno 113] No route to host',))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "hdfs_read.py", line 3, in <module>
with client.read('/home/edhuser/testdata.txt') as reader:
File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
return next(self.gen)
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 678, in read
buffersize=buffer_size,
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 118, in api_handler
raise err
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 107, in api_handler
**self.kwargs
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 207, in _request
**kwargs
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 468, in request
resp = self.send(prep, **send_kwargs)
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 576, in send
r = adapter.send(request, **kwargs)
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 437, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='192.168.12.52', port=9000): Max retries exceeded with url: /webhdfs/v1/home/edhuser/testdata.txt?user.name=embs&offset=0&op=OPEN (Caused by NewConnectionError('<requests.packages.urllib3.connection.HTTPConnection object at 0x7f2d88cef2b0>: Failed to establish a new connection: [Errno 113] No route to host',))
How can I fix this connection issue? Am I using a wrong port? I though the port that the namenode is using is specified in the core-site.xml, which I have shown above as stating 9000 for the port. In any case, I have tried all the default ports 50070, 8020, 8048 mentioned in the hadoop installation doc for various purposes, and I still get the same error. Instead of client = InsecureClient('http://192.168.12.52:9000'), should I be using client = InsecureClient('hdfs://192.168.12.52:9000'), or maybe client = InsecureClient('file:///192.168.12.52:9000'), or something like that? I have seen these elsewhere at various times.
I can access the HDFS in web by the way, as shown in the screenshot below:
Also, even if it connects successfully, I think I may not be giving the right file path (/opt/hadoop/README.txt). I gave this file path as this is what I see when I do search for the list of files and directories in the hadoop installation directory, which is /opt/hadoop:
$ ls /opt/hadoop/
bin lib read_from_hdfs.py write_to_hdfs_2.py
connect_to_hdfs.py libexec README.txt write_to_hdfs3.py
etc LICENSE.txt sbin write_to_hdfs.py
hdfs_read_write.py logs share
include NOTICE.txt test_storage
But I know that HDFS is separate, and maybe I copied the contents of my HDFS into the local machine by doing hdfs dfs -get /test_storage/ ./ before, which is why its showing these files. But when I search for the files in the path of the namenode, it returns some illegible files:
$ls /opt/volume/namenode/current/
edits_0000000000000000001-0000000000000000002
edits_0000000000000000003-0000000000000000010
edits_0000000000000000011-0000000000000000012
edits_0000000000000000013-0000000000000000015
edits_0000000000000000016-0000000000000000023
edits_0000000000000000024-0000000000000000025
edits_0000000000000000026-0000000000000000032
edits_0000000000000000033-0000000000000000033
edits_0000000000000000034-0000000000000000035
edits_0000000000000000036-0000000000000000037
edits_0000000000000000038-0000000000000000039
edits_0000000000000000040-0000000000000000041
edits_0000000000000000042-0000000000000000043
edits_0000000000000000044-0000000000000000045
edits_0000000000000000046-0000000000000000047
edits_0000000000000000048-0000000000000000049
edits_0000000000000000050-0000000000000000051
edits_0000000000000000052-0000000000000000053
edits_0000000000000000054-0000000000000000055
edits_0000000000000000056-0000000000000000057
edits_0000000000000000058-0000000000000000059
edits_0000000000000000060-0000000000000000061
edits_0000000000000000062-0000000000000000063
edits_0000000000000000064-0000000000000000065
edits_0000000000000000066-0000000000000000067
edits_0000000000000000068-0000000000000000070
edits_0000000000000000071-0000000000000000072
edits_0000000000000000073-0000000000000000074
edits_0000000000000000075-0000000000000000076
edits_0000000000000000077-0000000000000000078
edits_inprogress_0000000000000000079
fsimage_0000000000000000076
fsimage_0000000000000000076.md5
fsimage_0000000000000000078
fsimage_0000000000000000078.md5
seen_txid
VERSION
So, if I am specifying the file path to read wrongly, what is the correct file path to use?
EDIT: Upon changing the port to 50070 (i.e., client = InsecureClient('http://192.168.12.52:50070')), I get the following error:
$ python3 hdfs_read_local.py
Traceback (most recent call last):
File "hdfs_read.py", line 3, in <module>
with client.read('/opt/hadoop/LICENSE.txt') as reader:
File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
return next(self.gen)
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 678, in read
buffersize=buffer_size,
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 112, in api_handler
raise err
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 107, in api_handler
**self.kwargs
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 210, in _request
_on_error(response)
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 50, in _on_error
raise HdfsError(message, exception=exception)
hdfs.util.HdfsError: File /opt/hadoop/LICENSE.txt not found.
EDIT2: Upon modifying the file path from /opt/hadoop/LICENSE.txt
to /test_storage/LICENSE.txt, which seems to be the correct HDFS path, Iand running the python script, I get the following error:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 137, in _new_conn
(self.host, self.port), self.timeout, **extra_kw)
File "/usr/lib/python3/dist-packages/urllib3/util/connection.py", line 91, in create_connection
raise err
File "/usr/lib/python3/dist-packages/urllib3/util/connection.py", line 81, in create_connection
sock.connect(sa)
OSError: [Errno 113] No route to host
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 560, in urlopen
body=body, headers=headers)
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 354, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/lib/python3.6/http/client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1026, in _send_output
self.send(msg)
File "/usr/lib/python3.6/http/client.py", line 964, in send
self.connect()
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 162, in connect
conn = self._new_conn()
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 146, in _new_conn
self, "Failed to establish a new connection: %s" % e)
requests.packages.urllib3.exceptions.NewConnectionError: <requests.packages.urllib3.connection.HTTPConnection object at 0x7f2e87867400>: Failed to establish a new connection: [Errno 113] No route to host
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 376, in send
timeout=timeout
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 610, in urlopen
_stacktrace=sys.exc_info()[2])
File "/usr/lib/python3/dist-packages/urllib3/util/retry.py", line 273, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
requests.packages.urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='pr2.embs', port=50075): Max retries exceeded with url: /webhdfs/v1/test_storage/LICENSE.txt?op=OPEN&user.name=embs&namenoderpcaddress=192.168.12.52:9000&offset=0 (Caused by NewConnectionError('<requests.packages.urllib3.connection.HTTPConnection object at 0x7f2e87867400>: Failed to establish a new connection: [Errno 113] No route to host',))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "hdfs_read_local.py", line 3, in <module>
with client.read('/test_storage/LICENSE.txt') as reader:
File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
return next(self.gen)
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 678, in read
buffersize=buffer_size,
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 118, in api_handler
raise err
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 107, in api_handler
**self.kwargs
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 207, in _request
**kwargs
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 468, in request
resp = self.send(prep, **send_kwargs)
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 597, in send
history = [resp for resp in gen] if allow_redirects else []
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 597, in <listcomp>
history = [resp for resp in gen] if allow_redirects else []
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 195, in resolve_redirects
**adapter_kwargs
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 576, in send
r = adapter.send(request, **kwargs)
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 437, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='pr2.embs', port=50075): Max retries exceeded with url: /webhdfs/v1/test_storage/LICENSE.txt?op=OPEN&user.name=embs&namenoderpcaddress=192.168.12.52:9000&offset=0 (Caused by NewConnectionError('<requests.packages.urllib3.connection.HTTPConnection object at 0x7f2e87867400>: Failed to establish a new connection: [Errno 113] No route to host',))
As stated here link this python library is using webhdfs. If you want to test both the host and the file path are correct you can use the following command curl -i 'http://192.168.12.52:50070/webhdfs/v1/<PATH>?op=LISTSTATUS'. This will list a directory in hdfs. if you get that right you can use the same "config" in python.
from hdfs import InsecureClient
client = InsecureClient('http://192.168.12.52:50070')
with client.read('<hdfs_path>') as reader:
features = reader.read()
print(features)
http://192.168.12.52:9000
9000 is an RPC port. 50070 is the default HTTP WebHDFS port.
You might get No route to host if WebHDFS is disabled, or the datanode is not exposing port 50075 (the datanode http address) because it's down, or you changed that property
client.read('/opt/hadoop/LICENSE.txt')
You're running HDFS in pseudo distributed mode, but you're trying to read a local file. /opt does not exist in HDFS by default, and you've only ran a local ls... You should instead be using hadoop fs -ls /opt to see what files do exist at the path you're trying to open
But when I search for the files in the path of the namenode, it returns some illegible files:
Your files are not stored in the namenode... Their metadata is
Your files are stored in the datanode data directories, but as blocks, not as human-readable content
You can run this command to get a list of all the blocks and their locations
hdfs fsck /path/to/file.txt -files -blocks
There could be a problem with network configuration. Try this tweaked code for the time-being:
from hdfs import InsecureClient
client = InsecureClient('http://0.0.0.0:50070')
with client.read('/test-storage/LICENSE.txt') as reader:
features = reader.read()
print(features)
Read about IP Address 0.0.0.0
Hi I'm faced similar issue. It' looks like port is right. Im my case I was able to get list of dirs, but couldn't write any data. Issue was in my vpn which blocks some of the ports, and read and write uses different one.
I am using Python 3.4 on Windows 7, and I am trying to test if a proxy allows or denies connections to particular websites by using a python script.
I am using the code below:
from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPError, urllib
conf = "http://{}:{}#{}".format(login, password, proxy)
supp = urllib.request.ProxyHandler({"http": conf})
auth = urllib.request.HTTPBasicAuthHandler()
open = urllib.request.build_opener(supp, auth, urllib.request.HTTPHandler)
urllib.request.install_opener(open)
response = urlopen(Request("http://www.google.com"))
I get no errors when executing the code above, however as soon as I switch the URL to an HTTPS one (for example, https://www.google.com), I get the following error:
C:\Python34\python.exe test_url.py
Traceback (most recent call last):
File "C:\Python34\lib\urllib\request.py", line 1182, in do_open
h.request(req.get_method(), req.selector, req.data, headers)
File "C:\Python34\lib\http\client.py", line 1088, in request
self._send_request(method, url, body, headers)
File "C:\Python34\lib\http\client.py", line 1126, in _send_request
self.endheaders(body)
File "C:\Python34\lib\http\client.py", line 1084, in endheaders
self._send_output(message_body)
File "C:\Python34\lib\http\client.py", line 922, in _send_output
self.send(msg)
File "C:\Python34\lib\http\client.py", line 857, in send
self.connect()
File "C:\Python34\lib\http\client.py", line 1223, in connect
super().connect()
File "C:\Python34\lib\http\client.py", line 834, in connect
self.timeout, self.source_address)
File "C:\Python34\lib\socket.py", line 494, in create_connection
for res in getaddrinfo(host, port, 0, SOCK_STREAM):
File "C:\Python34\lib\socket.py", line 533, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno 11004] getaddrinfo failed
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "test.py", line 14, in <module>
response = urlopen(Request("https://www.google.com"))
File "C:\Python34\lib\urllib\request.py", line 161, in urlopen
return opener.open(url, data, timeout)
File "C:\Python34\lib\urllib\request.py", line 463, in open
response = self._open(req, data)
File "C:\Python34\lib\urllib\request.py", line 481, in _open
'_open', req)
File "C:\Python34\lib\urllib\request.py", line 441, in _call_chain
result = func(*args)
File "C:\Python34\lib\urllib\request.py", line 1225, in https_open
context=self._context, check_hostname=self._check_hostname)
File "C:\Python34\lib\urllib\request.py", line 1184, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error [Errno 11004] getaddrinfo failed>
Any ideas why my code is only working with HTTP websites?
You need to specify the proxy handler for HTTPS separately, as it is a different protocol to HTTP. So the ProxyHandler line should be changed to:
supp = urllib.request.ProxyHandler({"http": conf, "https": conf})