Develop Reference

how to download this zip file using python requests?

I am trying to download a zip file that is stored here:
http://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11/N45W074.SRTMGL1.hgt.zip
If you paste this into the browser and hit enter, it will download the .zip folder.
If you inspect the browser while this is happening, you will see that there is an internal redirect going on:
And eventually the zip gets downloaded.
I am trying to automate this downloading using the python requests library by doing the following:
import requests
requests.get(url,
allow_redirects=True,
headers={'User-Agent':'Chrome/107.0.0.0'})
I've tried tons of combinations, using the full header string from the HTML inspection, forcing verify=True, with and without redirects, adding a HTTPBasicAuth user/pass that says is required although the file seems to download fine without any credentials.
Honestly no clue what I'm missing, this is not my expertise. I keep getting this error:
>>> requests.get(url,
... allow_redirects=True,
... headers={'User-Agent':'Chrome/107.0.0.0'})
Traceback (most recent call last):
File "C:\Users\dere\Miniconda3\envs\blender\lib\site-packages\urllib3\connection.py", line 174, in _new_conn
conn = connection.create_connection(
File "C:\Users\dere\Miniconda3\envs\blender\lib\site-packages\urllib3\util\connection.py", line 95, in create_connection
raise err
File "C:\Users\dere\Miniconda3\envs\blender\lib\site-packages\urllib3\util\connection.py", line 85, in create_connection
sock.connect(sa)
ConnectionRefusedError: [WinError 10061] No connection could be made because the target machine
actively refused it
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\dere\Miniconda3\envs\blender\lib\site-packages\urllib3\connectionpool.py", line 703, in urlopen
httplib_response = self._make_request(
File "C:\Users\dere\Miniconda3\envs\blender\lib\site-packages\urllib3\connectionpool.py", line 398, in _make_request
conn.request(method, url, **httplib_request_kw)
File "C:\Users\dere\Miniconda3\envs\blender\lib\site-packages\urllib3\connection.py", line 239, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "C:\Users\dere\Miniconda3\envs\blender\lib\http\client.py", line 1282, in request
self._send_request(method, url, body, headers, encode_chunked)
File "C:\Users\dere\Miniconda3\envs\blender\lib\http\client.py", line 1328, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "C:\Users\dere\Miniconda3\envs\blender\lib\http\client.py", line 1277, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "C:\Users\dere\Miniconda3\envs\blender\lib\http\client.py", line 1037, in _send_output
self.send(msg)
File "C:\Users\dere\Miniconda3\envs\blender\lib\http\client.py", line 975, in send
self.connect()
File "C:\Users\dere\Miniconda3\envs\blender\lib\site-packages\urllib3\connection.py", line 205, in connect
conn = self._new_conn()
File "C:\Users\dere\Miniconda3\envs\blender\lib\site-packages\urllib3\connection.py", line 186, in _new_conn
raise NewConnectionError(
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x0000016B76A6FE50>: Failed to establish a new connection: [WinError 10061] No connection could be made because the target machine actively refused it
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\dere\Miniconda3\envs\blender\lib\site-packages\requests\adapters.py", line 489, in send
resp = conn.urlopen(
File "C:\Users\dere\Miniconda3\envs\blender\lib\site-packages\urllib3\connectionpool.py", line 787, in urlopen
retries = retries.increment(
File "C:\Users\dere\Miniconda3\envs\blender\lib\site-packages\urllib3\util\retry.py", line 592, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='e4ftl01.cr.usgs.gov', port=80): Max retries exceeded with url: /MEASURES/SRTMGL1.003/2000.02.11/N45W074.SRTMGL1.hgt.zip (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x0000016B76A6FE50>: Failed to establish a new connection: [WinError 10061] No connection could be made because the target machine actively refused it'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "C:\Users\dere\Miniconda3\envs\blender\lib\site-packages\requests\api.py", line 73, in get
return request("get", url, params=params, **kwargs)
File "C:\Users\dere\Miniconda3\envs\blender\lib\site-packages\requests\api.py", line 59, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Users\dere\Miniconda3\envs\blender\lib\site-packages\requests\sessions.py", line 587, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\dere\Miniconda3\envs\blender\lib\site-packages\requests\sessions.py", line 701, in send
r = adapter.send(request, **kwargs)
File "C:\Users\dere\Miniconda3\envs\blender\lib\site-packages\requests\adapters.py", line 565, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='e4ftl01.cr.usgs.gov', port=80): Max retries exceeded with url: /MEASURES/SRTMGL1.003/2000.02.11/N45W074.SRTMGL1.hgt.zip (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x0000016B76A6FE50>: Failed to establish a new connection: [WinError 10061] No connection could be made because the target machine actively refused it'))
Can someone help me arrive at the code that will result in a successful request? I know how to write the request into a zip afterwards..

Change http to Https: This should work
import requests
# download zip file from url
url = "https://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL1.003/2000.02.11/N45W074.SRTMGL1.hgt.zip"
r = requests.get(url)
with open("N45W074.SRTMGL1.hgt.zip", "wb") as f:
f.write(r.content)

Thanks to #cnemri for the tip.
It was a combination of changing http to https and also including this specific cookie header in the request header:
headers = {'Cookie': '_gid=GA1.2.48775707.1669266346; _hjSessionUser_606685=eyJpZCI6IjQ1MzEzM2QzLTI3MWEtNWM0YS04M2YzLWRmMmMzNDk4NjY1ZSIsImNyZWF0ZWQiOjE2NjkyNjYzNDU2MjUsImV4aXN0aW5nIjp0cnVlfQ==; ERS_production_2=b7dfade669180a6d6d250b030ffc3cf2UZZa8%2F471MfcV%2FaNuFtu6Pli%2BpP8jKOIwR4JvQjm%2B6DLPjl679vVf5SCDk7C5TLQsj7qckIev6lmtGb6Mes5RKDHUs%2BBp3EAjKW2%2BMCUpV%2Fnx0z1pdaCQQ%3D%3D; EROS_SSO_production_secure=eyJjcmVhdGVkIjoxNjY5MjY5MjEzLCJ1cGRhdGVkIjoiMjAyMi0xMS0yMyAyMzo1MzoyOCIsImF1dGhUeXBlIjoiRVJTIiwiYXV0aFNlcnZpY2UiOiJFUk9TIiwidmVyc2lvbiI6MS4xLCJzdGF0ZSI6ImVjMDY3YmMzYmRhMzBiZTUyNTkxYTNiZTYwMTMwZWNmNjAwMWU1Y2JlMGMxZmNkYTU4Y2Y4OTY0YjRlNTJkOTEiLCJpZCI6InY5U1Q4YVl3M1hRKCsyIiwic2VjcmV0IjoiPl8lMCZPYiY9MUVkclRLZTt4fHVZLVcyU1VvTiA1SE17KiQqaG12OSxvPl5%2BdyJ9; _ga_0YWDZEJ295=GS1.1.1669269458.1.0.1669269460.0.0.0; _ga=GA1.2.1055277358.1669266346; _ga_71JPYV1CCS=GS1.1.1669306300.2.1.1669306402.0.0.0; DATA=Y3_gg9uD6LmunsfDHneR9wAAARQ'}
then, this worked:
requests.get(url, headers=headers)
I'm not sure if this is the "solution" or just a workaround I've discovered. Still appreciate any input. I think it may just be credentials that are stored?

Cannot perform HTTP GET inside container from python script

I have a simple container running a python script which needs to perform a HTTP GET. Problem is, ping works fine, both resolving the name with mDNS as well as with standard IP.
However, the HTTP GET does not work and the script hangs on it. Clearly the same script outside the container works just fine. This is the errors I am getting:
>>> requests.get("http://iot-ns-controller.local:8081")
Traceback (most recent call last):
File "/usr/lib/python3.10/site-packages/urllib3/connection.py", line 174, in _new_conn
conn = connection.create_connection(
File "/usr/lib/python3.10/site-packages/urllib3/util/connection.py", line 95, in create_connection
raise err
File "/usr/lib/python3.10/site-packages/urllib3/util/connection.py", line 85, in create_connection
sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3.10/site-packages/urllib3/connectionpool.py", line 703, in urlopen
httplib_response = self._make_request(
File "/usr/lib/python3.10/site-packages/urllib3/connectionpool.py", line 398, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/lib/python3.10/site-packages/urllib3/connection.py", line 239, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "/usr/lib/python3.10/http/client.py", line 1282, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.10/http/client.py", line 1328, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.10/http/client.py", line 1277, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.10/http/client.py", line 1037, in _send_output
self.send(msg)
File "/usr/lib/python3.10/http/client.py", line 975, in send
self.connect()
File "/usr/lib/python3.10/site-packages/urllib3/connection.py", line 205, in connect
conn = self._new_conn()
File "/usr/lib/python3.10/site-packages/urllib3/connection.py", line 186, in _new_conn
raise NewConnectionError(
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x7faa6658a0>: Failed to establish a new connection: [Errno 111] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3.10/site-packages/requests/adapters.py", line 489, in send
resp = conn.urlopen(
File "/usr/lib/python3.10/site-packages/urllib3/connectionpool.py", line 787, in urlopen
retries = retries.increment(
File "/usr/lib/python3.10/site-packages/urllib3/util/retry.py", line 592, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='iot-ns-controller.local', port=8081): Max retries exceeded with url: / (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7faa6658a0>: Failed to establish a new connection: [Errno 111] Connection refused'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/lib/python3.10/site-packages/requests/api.py", line 73, in get
return request("get", url, params=params, **kwargs)
File "/usr/lib/python3.10/site-packages/requests/api.py", line 59, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/lib/python3.10/site-packages/requests/sessions.py", line 587, in request
resp = self.send(prep, **send_kwargs)
File "/usr/lib/python3.10/site-packages/requests/sessions.py", line 701, in send
r = adapter.send(request, **kwargs)
File "/usr/lib/python3.10/site-packages/requests/adapters.py", line 565, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='iot-ns-controller.local', port=8081): Max retries exceeded with url: / (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7faa6658a0>: Failed to establish a new connection: [Errno 111] Connection refused'))
It also does not work if I query host.docker.internal and I get the same error

"url = ( )" connection timeout when connected to proxied network

I have tried to find the right answer for my situation, but it seems it may not exists yet.
In my current training course (very beginner level) I have an exercise to get some weather json info from a url (Open Weather in this case). Currently, this is the only way I have been taught to do this:
url = "http://api.openweathermap.org/data/2.5/weather?q="+city+"&appid="+api_key
request = requests.get(url)
json = request.json()
print(json)
I have my "city" and "api_key" parameters set as instructed, and have been able to successfully run my code and get the json info when connected to my personal home network. However, when I connect to my work network, physically at the office or via VPN, I get this magical wall:
Traceback (most recent call last): File
"C:\Users\*REDACTED*\Python\Python39\lib\site-packages\urllib3\connection.py",
line 169, in _new_conn
conn = connection.create_connection( File "C:\Users\*REDACTED*\Python\Python39\lib\site-packages\urllib3\util\connection.py",
line 96, in create_connection
raise err File "C:\Users\*REDACTED*\Python\Python39\lib\site-packages\urllib3\util\connection.py",
line 86, in create_connection
sock.connect(sa) TimeoutError: [WinError 10060] A connection attempt failed because the connected party did not properly respond
after a period of time, or established connection failed because
connected host has failed to respond
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File
"C:\Users\*REDACTED*\Python\Python39\lib\site-packages\urllib3\connectionpool.py",
line 699, in urlopen
httplib_response = self._make_request( File "C:\Users\*REDACTED*\Python\Python39\lib\site-packages\urllib3\connectionpool.py",
line 394, in _make_request
conn.request(method, url, **httplib_request_kw) File "C:\Users\*REDACTED*\Python\Python39\lib\site-packages\urllib3\connection.py",
line 234, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers) File
"C:\Users\*REDACTED*\Python\Python39\lib\http\client.py", line 1257,
in request
self._send_request(method, url, body, headers, encode_chunked) File "C:\Users\*REDACTED*\Python\Python39\lib\http\client.py", line
1303, in _send_request
self.endheaders(body, encode_chunked=encode_chunked) File "C:\Users\*REDACTED*\Python\Python39\lib\http\client.py", line 1252,
in endheaders
self._send_output(message_body, encode_chunked=encode_chunked) File "C:\Users\*REDACTED*\Python\Python39\lib\http\client.py", line
1012, in _send_output
self.send(msg) File "C:\Users\*REDACTED*\Python\Python39\lib\http\client.py", line 952, in
send
self.connect() File "C:\Users\*REDACTED*\Python\Python39\lib\site-packages\urllib3\connection.py",
line 200, in connect
conn = self._new_conn() File "C:\Users\*REDACTED*\Python\Python39\lib\site-packages\urllib3\connection.py",
line 181, in _new_conn
raise NewConnectionError( urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x00000255A4B11370>:
Failed to establish a new connection: [WinError 10060] A connection
attempt failed because the connected party did not properly respond
after a period of time, or established connection failed because
connected host has failed to respond
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File
"C:\Users\*REDACTED*\Python\Python39\lib\site-packages\requests-2.26.0-py3.9.egg\requests\adapters.py",
line 439, in send
resp = conn.urlopen( File "C:\Users\*REDACTED*\Python\Python39\lib\site-packages\urllib3\connectionpool.py",
line 755, in urlopen
retries = retries.increment( File "C:\Users\*REDACTED*\Python\Python39\lib\site-packages\urllib3\util\retry.py",
line 574, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause)) urllib3.exceptions.MaxRetryError:
HTTPConnectionPool(host='api.openweathermap.org', port=80): Max
retries exceeded with url:
/data/2.5/weather?q=*REDACTED*&appid=*REDACTED* (Caused by
NewConnectionError('<urllib3.connection.HTTPConnection object at
0x00000255A4B11370>: Failed to establish a new connection: [WinError
10060] A connection attempt failed because the connected party did not
properly respond after a period of time, or established connection
failed because connected host has failed to respond'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File
"c:\Users\*REDACTED*\Training\Code\Python\weather.py", line 7, in
<module>
request = requests.get(url) File "C:\Users\*REDACTED*\Python\Python39\lib\site-packages\requests-2.26.0-py3.9.egg\requests\api.py",
line 75, in get
return request('get', url, params=params, **kwargs)
return session.request(method=method, url=url, **kwargs) File "C:\Users\*REDACTED*\Python\Python39\lib\site-packages\requests-2.26.0-py3.9.egg\requests\sessions.py",
line 542, in request
resp = self.send(prep, **send_kwargs) File "C:\Users\*REDACTED*\Python\Python39\lib\site-packages\requests-2.26.0-py3.9.egg\requests\sessions.py",
line 655, in send
r = adapter.send(request, **kwargs) File "C:\Users\*REDACTED*\Python\Python39\lib\site-packages\requests-2.26.0-py3.9.egg\requests\adapters.py",
line 516, in send
raise ConnectionError(e, request=request) requests.exceptions.ConnectionError:
HTTPConnectionPool(host='api.openweathermap.org', port=80): Max
retries exceeded with url:
/data/2.5/weather?q=*REDACTED*&appid=*REDACTED* (Caused by
NewConnectionError('<urllib3.connection.HTTPConnection object at
0x00000255A4B11370>: Failed to establish a new connection: [WinError
10060] A connection attempt failed because the connected party did not
properly respond after a period of time, or established connection
failed because connected host has failed to respond')) PS
C:\Users\*REDACTED*\Training\Code\Python&
C:/Users/*REDACTED*/Python/Python39/python.exe
"c:/Users/*REDACTED*/Training/Code/Python/weather.py" Traceback (most
recent call last): File
"C:\Users\*REDACTED*\Python\Python39\lib\site-packages\urllib3\connection.py",
line 169, in _new_conn
conn = connection.create_connection( File "C:\Users\*REDACTED*\Python\Python39\lib\site-packages\urllib3\util\connection.py",
line 96, in create_connection
raise err File "C:\Users\*REDACTED*\Python\Python39\lib\site-packages\urllib3\util\connection.py",
line 86, in create_connection
sock.connect(sa) TimeoutError: [WinError 10060] A connection attempt failed because the connected party did not properly respond
after a period of time, or established connection failed because
connected host has failed to respond
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File
"C:\Users\*REDACTED*\Python\Python39\lib\site-packages\urllib3\connectionpool.py",
line 699, in urlopen
httplib_response = self._make_request( File "C:\Users\*REDACTED*\Python\Python39\lib\site-packages\urllib3\connectionpool.py",
line 394, in _make_request
conn.request(method, url, **httplib_request_kw) File "C:\Users\*REDACTED*\Python\Python39\lib\site-packages\urllib3\connection.py",
line 234, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers) File
"C:\Users\*REDACTED*\Python\Python39\lib\http\client.py", line 1257,
in request
self._send_request(method, url, body, headers, encode_chunked) File "C:\Users\*REDACTED*\Python\Python39\lib\http\client.py", line
1303, in _send_request
self.endheaders(body, encode_chunked=encode_chunked) File "C:\Users\*REDACTED*\Python\Python39\lib\http\client.py", line 1252,
in endheaders
self._send_output(message_body, encode_chunked=encode_chunked) File "C:\Users\*REDACTED*\Python\Python39\lib\http\client.py", line
1012, in _send_output
self.send(msg) File "C:\Users\*REDACTED*\Python\Python39\lib\http\client.py", line 952, in
send
self.connect() File "C:\Users\*REDACTED*\Python\Python39\lib\site-packages\urllib3\connection.py",
line 200, in connect
conn = self._new_conn() File "C:\Users\*REDACTED*\Python\Python39\lib\site-packages\urllib3\connection.py",
line 181, in _new_conn
raise NewConnectionError( urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x00000229035A1370>:
Failed to establish a new connection: [WinError 10060] A connection
attempt failed because the connected party did not properly respond
after a period of time, or established connection failed because
connected host has failed to respond
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File
"C:\Users\*REDACTED*\Python\Python39\lib\site-packages\requests-2.26.0-py3.9.egg\requests\adapters.py",
line 439, in send
resp = conn.urlopen( File "C:\Users\*REDACTED*\Python\Python39\lib\site-packages\urllib3\connectionpool.py",
line 755, in urlopen
retries = retries.increment( File "C:\Users\*REDACTED*\Python\Python39\lib\site-packages\urllib3\util\retry.py",
line 574, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause)) urllib3.exceptions.MaxRetryError:
HTTPConnectionPool(host='api.openweathermap.org', port=80): Max
retries exceeded with url:
/data/2.5/weather?q=*REDACTED*&appid=*REDACTED* (Caused by
NewConnectionError('<urllib3.connection.HTTPConnection object at
0x00000229035A1370>: Failed to establish a new connection: [WinError
10060] A connection attempt failed because the connected party did not
properly respond after a period of time, or established connection
failed because connected host has failed to respond'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File
"c:\Users\*REDACTED*\Training\Code\Python\weather.py", line 7, in
<module>
request = requests.get(url) File "C:\Users\*REDACTED*\Python\Python39\lib\site-packages\requests-2.26.0-py3.9.egg\requests\api.py",
line 75, in get
return request('get', url, params=params, **kwargs) File "C:\Users\*REDACTED*\Python\Python39\lib\site-packages\requests-2.26.0-py3.9.egg\requests\api.py",
line 61, in request
return session.request(method=method, url=url, **kwargs) File "C:\Users\*REDACTED*\Python\Python39\lib\site-packages\requests-2.26.0-py3.9.egg\requests\sessions.py",
line 542, in request
resp = self.send(prep, **send_kwargs) File "C:\Users\*REDACTED*\Python\Python39\lib\site-packages\requests-2.26.0-py3.9.egg\requests\sessions.py",
line 655, in send
r = adapter.send(request, **kwargs) File "C:\Users\*REDACTED*\Python\Python39\lib\site-packages\requests-2.26.0-py3.9.egg\requests\adapters.py",
line 516, in send
raise ConnectionError(e, request=request) requests.exceptions.ConnectionError:
HTTPConnectionPool(host='api.openweathermap.org', port=80): Max
retries exceeded with url:
/data/2.5/weather?q=*REDACTED*&appid=*REDACTED* (Caused by
NewConnectionError('<urllib3.connection.HTTPConnection object at
0x00000229035A1370>: Failed to establish a new connection: [WinError
10060] A connection attempt failed because the connected party did not
properly respond after a period of time, or established connection
failed because connected host has failed to respond'))
Quick ref of facts/actions taken:
-Tried setting the proxy globally to all Python instances via the set proxy... command.
-Not using a venv.
-Tried to add the --proxy... argument at the end of the command before executing via the the Visual Studio Code terminal.
-On Windows 10 Enterprise.
I am unable to move to the next module of the course because I need this code to work so that I know how to fix the error in the next set of codes I will have to do. I also apologize if this is a lot of words to get through (I am a noob for a reason, I guess), but I really appreciate anyone who could guide me.

For Requests proxy set in own manner Proxies in Requests

Thanks to the help of a colleague, I was able to better interpret the link that Сергей Зайков referenced. It was a proxy issue after all.
After making my adjustment in the code, I was able to modify it by adding the lines below and updating the request.get
http_proxy = "http://*REDACTED*:8080"
proxyDict = {"http" : http_proxy}
request = requests.get(url, proxies=proxyDict)
I am now able to run my code as intended.

Python request fails from google cloud platform (app engine & cloud shell)

I have a very simple python http request which works on my local machine, but does not work in google app engine or from the google cloud shell. There must be config that is messing with the request, but I can't seem to find the problem.
I am using python 3.7
The code that causes the exception is the following (I have replaced the actual URL by example.com)
(The service I need to call is listening on port 6123)
import requests
myrequest = requests.get("http://example.com:6123/something?param=123")
The full error message I get is the following:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 672, in urlopen
chunked=chunked,
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 387, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/local/lib/python3.7/http/client.py", line 1229, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/local/lib/python3.7/http/client.py", line 1275, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/local/lib/python3.7/http/client.py", line 1224, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/local/lib/python3.7/http/client.py", line 1016, in _send_output
self.send(msg)
File "/usr/local/lib/python3.7/http/client.py", line 956, in send
self.connect()
File "/usr/local/lib/python3.7/site-packages/urllib3/connection.py", line 184, in connect
conn = self._new_conn()
File "/usr/local/lib/python3.7/site-packages/urllib3/connection.py", line 169, in _new_conn
self, "Failed to establish a new connection: %s" % e
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x7f0b242cd710>: Failed to establish a new connection: [Errno -2] Name or service not known
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 720, in urlopen
method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
File "/usr/local/lib/python3.7/site-packages/urllib3/util/retry.py", line 436, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='example.com', port=6123): Max retries exceeded with url: /something?param=123 (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f0b242cd710>: Failed to establish a new connection: [Errno -2] Name or service not known'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/local/lib/python3.7/site-packages/requests/api.py", line 75, in get
return request('get', url, params=params, **kwargs)
File "/usr/local/lib/python3.7/site-packages/requests/api.py", line 60, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/local/lib/python3.7/site-packages/requests/sessions.py", line 533, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python3.7/site-packages/requests/sessions.py", line 646, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python3.7/site-packages/requests/adapters.py", line 516, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='example.com', port=6123): Max retries exceeded with url: /something?param=123 (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f0b242cd710>: Failed to establish a new connection: [Errno -2] Name or service not known'))

Domain is not available from the outside, only from local network. Thanks Dustin Ingram!

Python automatic login with Requests and Beautifulsoup from list

trying to learn Python at the moment and am currently developing a small program which imports a list of login pages and tries your username/password automatically on them. Unfortunately I'm running into some errors and can't quite figure it out what the issue is. Please excuse the messy code by the way, still working on it.
Here's my source:
import requests
from bs4 import BeautifulSoup
import_file_path = input('Enter the path of the list to be tested: ')
export_file_path = input('Enter the path of where we should export the websites that worked to:: ')
with open(import_file_path, 'r') as panels:
panel_list = []
for line in panels:
panel_list.append(line)
x = 0
for panel in panel_list:
url = requests.get(panel)
soup = BeautifulSoup(url.content, "html.parser")
forms = soup.find_all("form")
action = soup.find('form').get('action')
values = {
soup.find_all("input")[0].get("name") : "user",
soup.find_all("input")[1].get("name") : "pass"
}
if "http://" or "https://" not in action:
action = 'http://' + action
r = requests.post(action, data=values)
print(r.content)
x += 1
Here are the errors I'm getting:
Enter the path of the website to be tested: list.txt
Enter the path of where we should export the vuln panels to: exit.txt
http://http://localhost/admin.php/vuln.php
Traceback (most recent call last):
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\urllib3\connection.py", line 141, in _new_conn
(self.host, self.port), self.timeout, **extra_kw)
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\urllib3\util\connection.py", line 60, in create_connection
for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\socket.py", line 743, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno 11001] getaddrinfo failed
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\urllib3\connectionpool.py", line 601, in urlopen
chunked=chunked)
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\urllib3\connectionpool.py", line 357, in _make_request
conn.request(method, url, **httplib_request_kw)
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 1026, in _send_output
self.send(msg)
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 964, in send
self.connect()
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\urllib3\connection.py", line 166, in connect
conn = self._new_conn()
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\urllib3\connection.py", line 150, in _new_conn
self, "Failed to establish a new connection: %s" % e)
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x038B9070>: Failed to establish a new connection: [Errno 11001] getaddrinfo failed
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\requests\adapters.py", line 440, in send
timeout=timeout
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\urllib3\connectionpool.py", line 639, in urlopen
_stacktrace=sys.exc_info()[2])
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\urllib3\util\retry.py", line 388, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='http', port=80): Max retries exceeded with url: //localhost/admin.php/vuln.php (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x038B9070>: Failed to establish a new connection: [Errno 11001] getaddrinfo failed',))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "scan.py", line 35, in <module>
r = requests.post(action, data=values)
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\requests\api.py", line 112, in post
return request('post', url, data=data, json=json, **kwargs)
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\requests\api.py", line 58, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\requests\sessions.py", line 508, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\requests\sessions.py", line 618, in send
r = adapter.send(request, **kwargs)
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\requests\adapters.py", line 508, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='http', port=80): Max retries exceeded with url: //localhost/admin.php/vuln.php (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x038B9070>: Failed to establish a new connection: [Errno 11001] getaddrinfo failed',))