Connection timeout error while reading a file from HDFS using Python - python
I have created a single node HDFS in a VM (hadoop.master, IP: 192.168.12.52). The file etc/hadoop/core-site.xml has the following configuration for the namenode:
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://master.hadoop:9000/</value>
</property>
</configuration>
I want to read a file from the HDFS on my local, physical desktop. For that, this is my code, which I've saved in a file named hdfs_read.py:
from hdfs import InsecureClient
client = InsecureClient('http://192.168.12.52:9000')
with client.read('/opt/hadoop/LICENSE.txt') as reader:
features = reader.read()
print(features)
Now when I run it, I get the following timeout error:
$ python3 hdfs_read.py
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 137, in _new_conn
(self.host, self.port), self.timeout, **extra_kw)
File "/usr/lib/python3/dist-packages/urllib3/util/connection.py", line 91, in create_connection
raise err
File "/usr/lib/python3/dist-packages/urllib3/util/connection.py", line 81, in create_connection
sock.connect(sa)
OSError: [Errno 113] No route to host
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 560, in urlopen
body=body, headers=headers)
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 354, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/lib/python3.6/http/client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1026, in _send_output
self.send(msg)
File "/usr/lib/python3.6/http/client.py", line 964, in send
self.connect()
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 162, in connect
conn = self._new_conn()
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 146, in _new_conn
self, "Failed to establish a new connection: %s" % e)
requests.packages.urllib3.exceptions.NewConnectionError: <requests.packages.urllib3.connection.HTTPConnection object at 0x7f2d88cef2b0>: Failed to establish a new connection: [Errno 113] No route to host
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 376, in send
timeout=timeout
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 610, in urlopen
_stacktrace=sys.exc_info()[2])
File "/usr/lib/python3/dist-packages/urllib3/util/retry.py", line 273, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
requests.packages.urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='192.168.12.52', port=9000): Max retries exceeded with url: /webhdfs/v1/home/edhuser/testdata.txt?user.name=embs&offset=0&op=OPEN (Caused by NewConnectionError('<requests.packages.urllib3.connection.HTTPConnection object at 0x7f2d88cef2b0>: Failed to establish a new connection: [Errno 113] No route to host',))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "hdfs_read_local.py", line 3, in <module>
with client.read('/home/edhuser/testdata.txt') as reader:
File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
return next(self.gen)
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 678, in read
buffersize=buffer_size,
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 118, in api_handler
raise err
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 107, in api_handler
**self.kwargs
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 207, in _request
**kwargs
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 468, in request
resp = self.send(prep, **send_kwargs)
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 576, in send
r = adapter.send(request, **kwargs)
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 437, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='192.168.12.52', port=9000): Max retries exceeded with url: /webhdfs/v1/home/edhuser/testdata.txt?user.name=embs&offset=0&op=OPEN (Caused by NewConnectionError('<requests.packages.urllib3.connection.HTTPConnection object at 0x7f2d88cef2b0>: Failed to establish a new connection: [Errno 113] No route to host',))
Error in sys.excepthook:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/apport_python_hook.py", line 63, in apport_excepthook
from apport.fileutils import likely_packaged, get_recent_crashes
File "/usr/lib/python3/dist-packages/apport/__init__.py", line 5, in <module>
from apport.report import Report
File "/usr/lib/python3/dist-packages/apport/report.py", line 30, in <module>
import apport.fileutils
File "/usr/lib/python3/dist-packages/apport/fileutils.py", line 23, in <module>
from apport.packaging_impl import impl as packaging
File "/usr/lib/python3/dist-packages/apport/packaging_impl.py", line 23, in <module>
import apt
File "/usr/lib/python3/dist-packages/apt/__init__.py", line 23, in <module>
import apt_pkg
ModuleNotFoundError: No module named 'apt_pkg'
Original exception was:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 137, in _new_conn
(self.host, self.port), self.timeout, **extra_kw)
File "/usr/lib/python3/dist-packages/urllib3/util/connection.py", line 91, in create_connection
raise err
File "/usr/lib/python3/dist-packages/urllib3/util/connection.py", line 81, in create_connection
sock.connect(sa)
OSError: [Errno 113] No route to host
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 560, in urlopen
body=body, headers=headers)
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 354, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/lib/python3.6/http/client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1026, in _send_output
self.send(msg)
File "/usr/lib/python3.6/http/client.py", line 964, in send
self.connect()
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 162, in connect
conn = self._new_conn()
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 146, in _new_conn
self, "Failed to establish a new connection: %s" % e)
requests.packages.urllib3.exceptions.NewConnectionError: <requests.packages.urllib3.connection.HTTPConnection object at 0x7f2d88cef2b0>: Failed to establish a new connection: [Errno 113] No route to host
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 376, in send
timeout=timeout
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 610, in urlopen
_stacktrace=sys.exc_info()[2])
File "/usr/lib/python3/dist-packages/urllib3/util/retry.py", line 273, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
requests.packages.urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='192.168.12.52', port=9000): Max retries exceeded with url: /webhdfs/v1/home/edhuser/testdata.txt?user.name=embs&offset=0&op=OPEN (Caused by NewConnectionError('<requests.packages.urllib3.connection.HTTPConnection object at 0x7f2d88cef2b0>: Failed to establish a new connection: [Errno 113] No route to host',))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "hdfs_read.py", line 3, in <module>
with client.read('/home/edhuser/testdata.txt') as reader:
File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
return next(self.gen)
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 678, in read
buffersize=buffer_size,
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 118, in api_handler
raise err
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 107, in api_handler
**self.kwargs
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 207, in _request
**kwargs
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 468, in request
resp = self.send(prep, **send_kwargs)
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 576, in send
r = adapter.send(request, **kwargs)
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 437, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='192.168.12.52', port=9000): Max retries exceeded with url: /webhdfs/v1/home/edhuser/testdata.txt?user.name=embs&offset=0&op=OPEN (Caused by NewConnectionError('<requests.packages.urllib3.connection.HTTPConnection object at 0x7f2d88cef2b0>: Failed to establish a new connection: [Errno 113] No route to host',))
How can I fix this connection issue? Am I using a wrong port? I though the port that the namenode is using is specified in the core-site.xml, which I have shown above as stating 9000 for the port. In any case, I have tried all the default ports 50070, 8020, 8048 mentioned in the hadoop installation doc for various purposes, and I still get the same error. Instead of client = InsecureClient('http://192.168.12.52:9000'), should I be using client = InsecureClient('hdfs://192.168.12.52:9000'), or maybe client = InsecureClient('file:///192.168.12.52:9000'), or something like that? I have seen these elsewhere at various times.
I can access the HDFS in web by the way, as shown in the screenshot below:
Also, even if it connects successfully, I think I may not be giving the right file path (/opt/hadoop/README.txt). I gave this file path as this is what I see when I do search for the list of files and directories in the hadoop installation directory, which is /opt/hadoop:
$ ls /opt/hadoop/
bin lib read_from_hdfs.py write_to_hdfs_2.py
connect_to_hdfs.py libexec README.txt write_to_hdfs3.py
etc LICENSE.txt sbin write_to_hdfs.py
hdfs_read_write.py logs share
include NOTICE.txt test_storage
But I know that HDFS is separate, and maybe I copied the contents of my HDFS into the local machine by doing hdfs dfs -get /test_storage/ ./ before, which is why its showing these files. But when I search for the files in the path of the namenode, it returns some illegible files:
$ls /opt/volume/namenode/current/
edits_0000000000000000001-0000000000000000002
edits_0000000000000000003-0000000000000000010
edits_0000000000000000011-0000000000000000012
edits_0000000000000000013-0000000000000000015
edits_0000000000000000016-0000000000000000023
edits_0000000000000000024-0000000000000000025
edits_0000000000000000026-0000000000000000032
edits_0000000000000000033-0000000000000000033
edits_0000000000000000034-0000000000000000035
edits_0000000000000000036-0000000000000000037
edits_0000000000000000038-0000000000000000039
edits_0000000000000000040-0000000000000000041
edits_0000000000000000042-0000000000000000043
edits_0000000000000000044-0000000000000000045
edits_0000000000000000046-0000000000000000047
edits_0000000000000000048-0000000000000000049
edits_0000000000000000050-0000000000000000051
edits_0000000000000000052-0000000000000000053
edits_0000000000000000054-0000000000000000055
edits_0000000000000000056-0000000000000000057
edits_0000000000000000058-0000000000000000059
edits_0000000000000000060-0000000000000000061
edits_0000000000000000062-0000000000000000063
edits_0000000000000000064-0000000000000000065
edits_0000000000000000066-0000000000000000067
edits_0000000000000000068-0000000000000000070
edits_0000000000000000071-0000000000000000072
edits_0000000000000000073-0000000000000000074
edits_0000000000000000075-0000000000000000076
edits_0000000000000000077-0000000000000000078
edits_inprogress_0000000000000000079
fsimage_0000000000000000076
fsimage_0000000000000000076.md5
fsimage_0000000000000000078
fsimage_0000000000000000078.md5
seen_txid
VERSION
So, if I am specifying the file path to read wrongly, what is the correct file path to use?
EDIT: Upon changing the port to 50070 (i.e., client = InsecureClient('http://192.168.12.52:50070')), I get the following error:
$ python3 hdfs_read_local.py
Traceback (most recent call last):
File "hdfs_read.py", line 3, in <module>
with client.read('/opt/hadoop/LICENSE.txt') as reader:
File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
return next(self.gen)
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 678, in read
buffersize=buffer_size,
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 112, in api_handler
raise err
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 107, in api_handler
**self.kwargs
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 210, in _request
_on_error(response)
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 50, in _on_error
raise HdfsError(message, exception=exception)
hdfs.util.HdfsError: File /opt/hadoop/LICENSE.txt not found.
EDIT2: Upon modifying the file path from /opt/hadoop/LICENSE.txt
to /test_storage/LICENSE.txt, which seems to be the correct HDFS path, Iand running the python script, I get the following error:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 137, in _new_conn
(self.host, self.port), self.timeout, **extra_kw)
File "/usr/lib/python3/dist-packages/urllib3/util/connection.py", line 91, in create_connection
raise err
File "/usr/lib/python3/dist-packages/urllib3/util/connection.py", line 81, in create_connection
sock.connect(sa)
OSError: [Errno 113] No route to host
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 560, in urlopen
body=body, headers=headers)
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 354, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/lib/python3.6/http/client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.6/http/client.py", line 1026, in _send_output
self.send(msg)
File "/usr/lib/python3.6/http/client.py", line 964, in send
self.connect()
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 162, in connect
conn = self._new_conn()
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 146, in _new_conn
self, "Failed to establish a new connection: %s" % e)
requests.packages.urllib3.exceptions.NewConnectionError: <requests.packages.urllib3.connection.HTTPConnection object at 0x7f2e87867400>: Failed to establish a new connection: [Errno 113] No route to host
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 376, in send
timeout=timeout
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 610, in urlopen
_stacktrace=sys.exc_info()[2])
File "/usr/lib/python3/dist-packages/urllib3/util/retry.py", line 273, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
requests.packages.urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='pr2.embs', port=50075): Max retries exceeded with url: /webhdfs/v1/test_storage/LICENSE.txt?op=OPEN&user.name=embs&namenoderpcaddress=192.168.12.52:9000&offset=0 (Caused by NewConnectionError('<requests.packages.urllib3.connection.HTTPConnection object at 0x7f2e87867400>: Failed to establish a new connection: [Errno 113] No route to host',))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "hdfs_read_local.py", line 3, in <module>
with client.read('/test_storage/LICENSE.txt') as reader:
File "/usr/lib/python3.6/contextlib.py", line 81, in __enter__
return next(self.gen)
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 678, in read
buffersize=buffer_size,
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 118, in api_handler
raise err
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 107, in api_handler
**self.kwargs
File "/home/embs/.local/lib/python3.6/site-packages/hdfs/client.py", line 207, in _request
**kwargs
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 468, in request
resp = self.send(prep, **send_kwargs)
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 597, in send
history = [resp for resp in gen] if allow_redirects else []
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 597, in <listcomp>
history = [resp for resp in gen] if allow_redirects else []
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 195, in resolve_redirects
**adapter_kwargs
File "/usr/lib/python3/dist-packages/requests/sessions.py", line 576, in send
r = adapter.send(request, **kwargs)
File "/usr/lib/python3/dist-packages/requests/adapters.py", line 437, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='pr2.embs', port=50075): Max retries exceeded with url: /webhdfs/v1/test_storage/LICENSE.txt?op=OPEN&user.name=embs&namenoderpcaddress=192.168.12.52:9000&offset=0 (Caused by NewConnectionError('<requests.packages.urllib3.connection.HTTPConnection object at 0x7f2e87867400>: Failed to establish a new connection: [Errno 113] No route to host',))
As stated here link this python library is using webhdfs. If you want to test both the host and the file path are correct you can use the following command curl -i 'http://192.168.12.52:50070/webhdfs/v1/<PATH>?op=LISTSTATUS'. This will list a directory in hdfs. if you get that right you can use the same "config" in python.
from hdfs import InsecureClient
client = InsecureClient('http://192.168.12.52:50070')
with client.read('<hdfs_path>') as reader:
features = reader.read()
print(features)
http://192.168.12.52:9000
9000 is an RPC port. 50070 is the default HTTP WebHDFS port.
You might get No route to host if WebHDFS is disabled, or the datanode is not exposing port 50075 (the datanode http address) because it's down, or you changed that property
client.read('/opt/hadoop/LICENSE.txt')
You're running HDFS in pseudo distributed mode, but you're trying to read a local file. /opt does not exist in HDFS by default, and you've only ran a local ls... You should instead be using hadoop fs -ls /opt to see what files do exist at the path you're trying to open
But when I search for the files in the path of the namenode, it returns some illegible files:
Your files are not stored in the namenode... Their metadata is
Your files are stored in the datanode data directories, but as blocks, not as human-readable content
You can run this command to get a list of all the blocks and their locations
hdfs fsck /path/to/file.txt -files -blocks
There could be a problem with network configuration. Try this tweaked code for the time-being:
from hdfs import InsecureClient
client = InsecureClient('http://0.0.0.0:50070')
with client.read('/test-storage/LICENSE.txt') as reader:
features = reader.read()
print(features)
Read about IP Address 0.0.0.0
Hi I'm faced similar issue. It' looks like port is right. Im my case I was able to get list of dirs, but couldn't write any data. Issue was in my vpn which blocks some of the ports, and read and write uses different one.
Related
Cannot perform HTTP GET inside container from python script
I have a simple container running a python script which needs to perform a HTTP GET. Problem is, ping works fine, both resolving the name with mDNS as well as with standard IP. However, the HTTP GET does not work and the script hangs on it. Clearly the same script outside the container works just fine. This is the errors I am getting: >>> requests.get("http://iot-ns-controller.local:8081") Traceback (most recent call last): File "/usr/lib/python3.10/site-packages/urllib3/connection.py", line 174, in _new_conn conn = connection.create_connection( File "/usr/lib/python3.10/site-packages/urllib3/util/connection.py", line 95, in create_connection raise err File "/usr/lib/python3.10/site-packages/urllib3/util/connection.py", line 85, in create_connection sock.connect(sa) ConnectionRefusedError: [Errno 111] Connection refused During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/lib/python3.10/site-packages/urllib3/connectionpool.py", line 703, in urlopen httplib_response = self._make_request( File "/usr/lib/python3.10/site-packages/urllib3/connectionpool.py", line 398, in _make_request conn.request(method, url, **httplib_request_kw) File "/usr/lib/python3.10/site-packages/urllib3/connection.py", line 239, in request super(HTTPConnection, self).request(method, url, body=body, headers=headers) File "/usr/lib/python3.10/http/client.py", line 1282, in request self._send_request(method, url, body, headers, encode_chunked) File "/usr/lib/python3.10/http/client.py", line 1328, in _send_request self.endheaders(body, encode_chunked=encode_chunked) File "/usr/lib/python3.10/http/client.py", line 1277, in endheaders self._send_output(message_body, encode_chunked=encode_chunked) File "/usr/lib/python3.10/http/client.py", line 1037, in _send_output self.send(msg) File "/usr/lib/python3.10/http/client.py", line 975, in send self.connect() File "/usr/lib/python3.10/site-packages/urllib3/connection.py", line 205, in connect conn = self._new_conn() File "/usr/lib/python3.10/site-packages/urllib3/connection.py", line 186, in _new_conn raise NewConnectionError( urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x7faa6658a0>: Failed to establish a new connection: [Errno 111] Connection refused During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/lib/python3.10/site-packages/requests/adapters.py", line 489, in send resp = conn.urlopen( File "/usr/lib/python3.10/site-packages/urllib3/connectionpool.py", line 787, in urlopen retries = retries.increment( File "/usr/lib/python3.10/site-packages/urllib3/util/retry.py", line 592, in increment raise MaxRetryError(_pool, url, error or ResponseError(cause)) urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='iot-ns-controller.local', port=8081): Max retries exceeded with url: / (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7faa6658a0>: Failed to establish a new connection: [Errno 111] Connection refused')) During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<stdin>", line 1, in <module> File "/usr/lib/python3.10/site-packages/requests/api.py", line 73, in get return request("get", url, params=params, **kwargs) File "/usr/lib/python3.10/site-packages/requests/api.py", line 59, in request return session.request(method=method, url=url, **kwargs) File "/usr/lib/python3.10/site-packages/requests/sessions.py", line 587, in request resp = self.send(prep, **send_kwargs) File "/usr/lib/python3.10/site-packages/requests/sessions.py", line 701, in send r = adapter.send(request, **kwargs) File "/usr/lib/python3.10/site-packages/requests/adapters.py", line 565, in send raise ConnectionError(e, request=request) requests.exceptions.ConnectionError: HTTPConnectionPool(host='iot-ns-controller.local', port=8081): Max retries exceeded with url: / (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7faa6658a0>: Failed to establish a new connection: [Errno 111] Connection refused')) It also does not work if I query host.docker.internal and I get the same error
requests.exceptions.ConnectionError: HTTPConnectionPool(host='localhost', port=5000): Max retries exceeded with url: /blockchain (Caused by NewConnect
enter image description herethis is my code where i get stuck Traceback (most recent call last): File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connection.py", line 159, in _new_conn conn = connection.create_connection( File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\util\connection.py", line 84, in create_connection raise err File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\util\connection.py", line 74, in create_connection sock.connect(sa) ConnectionRefusedError: [WinError 10061] No connection could be made because the target machine actively refused it During handling of the above exception, another exception occurred: Traceback (most recent call last): File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connectionpool.py", line 670, in urlopen httplib_response = self._make_request( File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connectionpool.py", line 392, in _make_request conn.request(method, url, **httplib_request_kw) File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.752.0_x64__qbz5n2kfra8p0\lib\http\client.py", line 1282, in request self._send_request(method, url, body, headers, encode_chunked) File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.752.0_x64__qbz5n2kfra8p0\lib\http\client.py", line 1328, in _send_request self.endheaders(body, encode_chunked=encode_chunked) File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.752.0_x64__qbz5n2kfra8p0\lib\http\client.py", line 1277, in endheaders self._send_output(message_body, encode_chunked=encode_chunked) File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.752.0_x64__qbz5n2kfra8p0\lib\http\client.py", line 1037, in _send_output self.send(msg) File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.752.0_x64__qbz5n2kfra8p0\lib\http\client.py", line 975, in send self.connect() File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connection.py", line 187, in connect conn = self._new_conn() File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connection.py", line 171, in _new_conn raise NewConnectionError( urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x000001AD66840430>: Failed to establish a new connection: [WinError 10061] No connection could be made because the target machine actively refused it During handling of the above exception, another exception occurred: Traceback (most recent call last): File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\adapters.py", line 439, in send resp = conn.urlopen( File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connectionpool.py", line 726, in urlopen retries = retries.increment( File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\util\retry.py", line 446, in increment raise MaxRetryError(_pool, url, error or ResponseError(cause)) urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='localhost', port=5000): Max retries exceeded with url: /blockchain (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x000001AD66840430>: Failed to establish a new connection: [WinError 10061] No connection could be made because the target machine actively refused it')) During handling of the above exception, another exception occurred: Traceback (most recent call last): File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.752.0_x64__qbz5n2kfra8p0\lib\runpy.py", line 187, in _run_module_as_main mod_name, mod_spec, code = _get_module_details(mod_name, _Error) File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.752.0_x64__qbz5n2kfra8p0\lib\runpy.py", line 146, in _get_module_details return _get_module_details(pkg_main_name, error) File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.752.0_x64__qbz5n2kfra8p0\lib\runpy.py", line 110, in _get_module_details __import__(pkg_name) File "C:\Users\annun\python-blockchain\backend\app\__init__.py", line 82, in <module> result = requests.get(f'http://localhost:{ROOT_PORT}/blockchain') File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\api.py", line 75, in get return request('get', url, params=params, **kwargs) File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\api.py", line 60, in request return session.request(method=method, url=url, **kwargs) File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\sessions.py", line 533, in request resp = self.send(prep, **send_kwargs) File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\sessions.py", line 646, in send r = adapter.send(request, **kwargs) File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\adapters.py", line 516, in send raise ConnectionError(e, request=request) requests.exceptions.ConnectionError: HTTPConnectionPool(host='localhost', port=5000): Max retries exceeded with url: /blockchain (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x000001AD66840430>: Failed to establish a new connection: [WinError 10061] No connection could be made because the target machine actively refused it'))
Post your code, with your question. Not the error report, we can't do anything with just the error report. It's not OUR code you're reporting on. It does however look like you might've entered the wrong domain/IP to connect to, but since it's localhost you've probably misconfigured it.
Ibis create impala table with pandas dataframe and get [Error 61] Connection refused
After doing impyla sql statement, I convert the results into pandas dataframe format. But now I want to auto create a temporary table on impala using Apache Ibis to create table and load a dataframe into it. The following codes are divided into 3 phase: phase 1 creates a null table with user-defined schema phase 2 creates a table which schema and data are came from another table phase 3 is the main part I want to do (suppose the dataframe is given) and error code shows below. Does anyone know what happened it is? Cuz the configurations are the same, I don't know why it will get connection exception when doing phase 3. Thank you a lot ! import ibis import os import pandas as pd hdfs_host = os.environ.get('IBIS_TEST_NN_HOST', 'xx.xx.xx.xx') hdfs_port = int(os.environ.get('IBIS_TEST_NN_PORT', 50070)) impala_host = os.environ.get('IBIS_TEST_IMPALA_HOST', 'xx.xx.xx.xx') impala_port = int(os.environ.get('IBIS_TEST_IMPALA_PORT', 21050)) hdfs = ibis.hdfs_connect(host=hdfs_host, port=hdfs_port) client = ibis.impala.connect( host=impala_host, port=impala_port, hdfs_client=hdfs, ) print(hdfs) # doctest: +ELLIPSIS print(client) # doctest: +ELLIPSIS db=client.database('tmp') table = db.test_table schema = table.schema() --------------------------------------------------------------------------phase 1 table_name = 'tmp.test1' if not client.exists_table(table_name): db.create_table(table_name, schema=schema) --------------------------------------------------------------------------phase 2 table_name = 'tmp.test2' expr = table.group_by('id').size() if not client.exists_table(table_name): db.create_table(table_name,expr,format='parquet') --------------------------------------------------------------------------phase 3 data = pd.DataFrame({'foo': [1, 2, 3, 4], 'bar': ['a', 'b', 'c', 'd']}) table_name = 'tmp.test3' if not client.exists_table(table_name): db.create_table(table_name, data) And the results show that : <ibis.filesystems.WebHDFS object at 0x10f2195f8> <ibis.impala.client.ImpalaClient object at 0x10f201860> Traceback (most recent call last): File "/Users/xy.l./.local/share/virtualenvs/Import_table-UzARfE0y/lib/python3.6/site-packages/urllib3/connection.py", line 160, in _new_conn (self._dns_host, self.port), self.timeout, **extra_kw File "/Users/xy.l./.local/share/virtualenvs/Import_table-UzARfE0y/lib/python3.6/site-packages/urllib3/util/connection.py", line 84, in create_connection raise err File "/Users/xy.l./.local/share/virtualenvs/Import_table-UzARfE0y/lib/python3.6/site-packages/urllib3/util/connection.py", line 74, in create_connection sock.connect(sa) ConnectionRefusedError: [Errno 61] Connection refused During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/Users/xy.l./.local/share/virtualenvs/Import_table-UzARfE0y/lib/python3.6/site-packages/urllib3/connectionpool.py", line 677, in urlopen chunked=chunked, File "/Users/xy.l./.local/share/virtualenvs/Import_table-UzARfE0y/lib/python3.6/site-packages/urllib3/connectionpool.py", line 392, in _make_request conn.request(method, url, **httplib_request_kw) File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1239, in request self._send_request(method, url, body, headers, encode_chunked) File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1285, in _send_request self.endheaders(body, encode_chunked=encode_chunked) File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1234, in endheaders self._send_output(message_body, encode_chunked=encode_chunked) File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 1026, in _send_output self.send(msg) File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py", line 964, in send self.connect() File "/Users/xy.l./.local/share/virtualenvs/Import_table-UzARfE0y/lib/python3.6/site-packages/urllib3/connection.py", line 187, in connect conn = self._new_conn() File "/Users/xy.l./.local/share/virtualenvs/Import_table-UzARfE0y/lib/python3.6/site-packages/urllib3/connection.py", line 172, in _new_conn self, "Failed to establish a new connection: %s" % e urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x10f26c2b0>: Failed to establish a new connection: [Errno 61] Connection refused During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/Users/xy.l./.local/share/virtualenvs/Import_table-UzARfE0y/lib/python3.6/site-packages/requests/adapters.py", line 449, in send timeout=timeout File "/Users/xy.l./.local/share/virtualenvs/Import_table-UzARfE0y/lib/python3.6/site-packages/urllib3/connectionpool.py", line 725, in urlopen method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2] File "/Users/xy.l./.local/share/virtualenvs/Import_table-UzARfE0y/lib/python3.6/site-packages/urllib3/util/retry.py", line 439, in increment raise MaxRetryError(_pool, url, error or ResponseError(cause)) urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='xx.xx.xx.xx', port=50070): Max retries exceeded with url: /webhdfs/v1/tmp/ibis/pandas_600666daa0c247daba610fd5b640525e?user.name=xy.l.&op=MKDIRS (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x10f26c2b0>: Failed to establish a new connection: [Errno 61] Connection refused',)) During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/Users/xy.l./Import_table/test.py", line 37, in <module> db.create_table(table_name, data) File "/Users/xy.l./.local/share/virtualenvs/Import_table-UzARfE0y/lib/python3.6/site-packages/ibis/impala/client.py", line 39, in create_table table_name, obj=obj, database=self.name, **kwargs File "/Users/xy.l./.local/share/virtualenvs/Import_table-UzARfE0y/lib/python3.6/site-packages/ibis/impala/client.py", line 1171, in create_table writer, to_insert = write_temp_dataframe(self, obj) File "/Users/xy.l./.local/share/virtualenvs/Import_table-UzARfE0y/lib/python3.6/site-packages/ibis/impala/pandas_interop.py", line 117, in write_temp_dataframe path = writer.write_temp_csv() File "/Users/xy.l./.local/share/virtualenvs/Import_table-UzARfE0y/lib/python3.6/site-packages/ibis/impala/pandas_interop.py", line 45, in write_temp_csv self.hdfs.mkdir(temp_hdfs_dir) File "/Users/xy.l./.local/share/virtualenvs/Import_table-UzARfE0y/lib/python3.6/site-packages/ibis/filesystems.py", line 511, in mkdir self.client.makedirs(dir_path) File "/Users/xy.l./.local/share/virtualenvs/Import_table-UzARfE0y/lib/python3.6/site-packages/hdfs/client.py", line 986, in makedirs self._mkdirs(hdfs_path, permission=permission) File "/Users/xy.l./.local/share/virtualenvs/Import_table-UzARfE0y/lib/python3.6/site-packages/hdfs/client.py", line 125, in api_handler raise err File "/Users/xy.l./.local/share/virtualenvs/Import_table-UzARfE0y/lib/python3.6/site-packages/hdfs/client.py", line 107, in api_handler **self.kwargs File "/Users/xy.l./.local/share/virtualenvs/Import_table-UzARfE0y/lib/python3.6/site-packages/hdfs/client.py", line 214, in _request **kwargs File "/Users/xy.l./.local/share/virtualenvs/Import_table-UzARfE0y/lib/python3.6/site-packages/requests/sessions.py", line 530, in request resp = self.send(prep, **send_kwargs) File "/Users/xy.l./.local/share/virtualenvs/Import_table-UzARfE0y/lib/python3.6/site-packages/requests/sessions.py", line 643, in send r = adapter.send(request, **kwargs) File "/Users/xy.l./.local/share/virtualenvs/Import_table-UzARfE0y/lib/python3.6/site-packages/requests/adapters.py", line 516, in send raise ConnectionError(e, request=request) requests.exceptions.ConnectionError: HTTPConnectionPool(host='xx.xx.xx.xx', port=50070): Max retries exceeded with url: /webhdfs/v1/tmp/ibis/pandas_600666daa0c247daba610fd5b640525e?user.name=xy.l.&op=MKDIRS (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x10f26c2b0>: Failed to establish a new connection: [Errno 61] Connection refused',)) Regards, XY.Ltw
Solved. Our environment is CDH6.3.2 and I check the Namenode web ui port dfs.namenode.http-address on CM is 9870 instead of 50070. Change hdfs client connection conf. on the code above and it will work well
Python request fails from google cloud platform (app engine & cloud shell)
I have a very simple python http request which works on my local machine, but does not work in google app engine or from the google cloud shell. There must be config that is messing with the request, but I can't seem to find the problem. I am using python 3.7 The code that causes the exception is the following (I have replaced the actual URL by example.com) (The service I need to call is listening on port 6123) import requests myrequest = requests.get("http://example.com:6123/something?param=123") The full error message I get is the following: Traceback (most recent call last): File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 672, in urlopen chunked=chunked, File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 387, in _make_request conn.request(method, url, **httplib_request_kw) File "/usr/local/lib/python3.7/http/client.py", line 1229, in request self._send_request(method, url, body, headers, encode_chunked) File "/usr/local/lib/python3.7/http/client.py", line 1275, in _send_request self.endheaders(body, encode_chunked=encode_chunked) File "/usr/local/lib/python3.7/http/client.py", line 1224, in endheaders self._send_output(message_body, encode_chunked=encode_chunked) File "/usr/local/lib/python3.7/http/client.py", line 1016, in _send_output self.send(msg) File "/usr/local/lib/python3.7/http/client.py", line 956, in send self.connect() File "/usr/local/lib/python3.7/site-packages/urllib3/connection.py", line 184, in connect conn = self._new_conn() File "/usr/local/lib/python3.7/site-packages/urllib3/connection.py", line 169, in _new_conn self, "Failed to establish a new connection: %s" % e urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x7f0b242cd710>: Failed to establish a new connection: [Errno -2] Name or service not known During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/local/lib/python3.7/site-packages/requests/adapters.py", line 449, in send timeout=timeout File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 720, in urlopen method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2] File "/usr/local/lib/python3.7/site-packages/urllib3/util/retry.py", line 436, in increment raise MaxRetryError(_pool, url, error or ResponseError(cause)) urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='example.com', port=6123): Max retries exceeded with url: /something?param=123 (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f0b242cd710>: Failed to establish a new connection: [Errno -2] Name or service not known')) During handling of the above exception, another exception occurred: Traceback (most recent call last): File "<stdin>", line 1, in <module> File "/usr/local/lib/python3.7/site-packages/requests/api.py", line 75, in get return request('get', url, params=params, **kwargs) File "/usr/local/lib/python3.7/site-packages/requests/api.py", line 60, in request return session.request(method=method, url=url, **kwargs) File "/usr/local/lib/python3.7/site-packages/requests/sessions.py", line 533, in request resp = self.send(prep, **send_kwargs) File "/usr/local/lib/python3.7/site-packages/requests/sessions.py", line 646, in send r = adapter.send(request, **kwargs) File "/usr/local/lib/python3.7/site-packages/requests/adapters.py", line 516, in send raise ConnectionError(e, request=request) requests.exceptions.ConnectionError: HTTPConnectionPool(host='example.com', port=6123): Max retries exceeded with url: /something?param=123 (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f0b242cd710>: Failed to establish a new connection: [Errno -2] Name or service not known'))
Domain is not available from the outside, only from local network. Thanks Dustin Ingram!
django-elasticsearch error : [Errno -2] Name or service not known
I'm trying to install an application developed by someone else and when running a manage.py command I'm getting this error, which I do not understand. I wonder if anyone can tell me where to look to investigate first - thank you. POST http://elasticsearch:9200/_bulk?refresh=true [status:N/A request:0.001s] Traceback (most recent call last): File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/django/db/models/query.py", line 486, in get_or_create return self.get(**lookup), False File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/djmoney/models/managers.py", line 209, in wrapper queryset = func(*args, **kwargs) File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/django/db/models/query.py", line 399, in get self.model._meta.object_name providers.models.Organisation.DoesNotExist: Organisation matching query does not exist. During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/connection.py", line 160, in _new_conn (self._dns_host, self.port), self.timeout, **extra_kw) File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/util/connection.py", line 57, in create_connection for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM): File "/usr/lib/python3.6/socket.py", line 745, in getaddrinfo for res in _socket.getaddrinfo(host, port, family, type, proto, flags): socket.gaierror: [Errno -2] Name or service not known During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/elasticsearch/connection/http_urllib3.py", line 95, in perform_request response = self.pool.urlopen(method, url, body, retries=False, headers=self.headers, **kw) File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/connectionpool.py", line 641, in urlopen _stacktrace=sys.exc_info()[2]) File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/util/retry.py", line 344, in increment raise six.reraise(type(error), error, _stacktrace) File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/packages/six.py", line 686, in reraise raise value File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/connectionpool.py", line 603, in urlopen chunked=chunked) File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/connectionpool.py", line 355, in _make_request conn.request(method, url, **httplib_request_kw) File "/usr/lib/python3.6/http/client.py", line 1254, in request self._send_request(method, url, body, headers, encode_chunked) File "/usr/lib/python3.6/http/client.py", line 1300, in _send_request self.endheaders(body, encode_chunked=encode_chunked) File "/usr/lib/python3.6/http/client.py", line 1249, in endheaders self._send_output(message_body, encode_chunked=encode_chunked) File "/usr/lib/python3.6/http/client.py", line 1036, in _send_output self.send(msg) File "/usr/lib/python3.6/http/client.py", line 974, in send self.connect() File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/connection.py", line 183, in connect conn = self._new_conn() File "/home/henry/.virtualenvs/hjsm/lib/python3.6/site-packages/urllib3/connection.py", line 169, in _new_conn self, "Failed to establish a new connection: %s" % e) urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x7f506c988eb8>: Failed to establish a new connection: [Errno -2] Name or service not known EDIT Added manage.py as requested #!/usr/bin/env python import os import sys if __name__ == "__main__": os.environ.setdefault('DEPLOY_ENV', 'local') os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings.%s' % os.environ['DEPLOY_ENV']) from django.core.management import execute_from_command_line # noqa execute_from_command_line(sys.argv)
As Goran said, I was trying to connect to elasticsearch:9200 but testing localhost:9200