Cannot perform HTTP GET inside container from python script - python

I have a simple container running a python script which needs to perform a HTTP GET. Problem is, ping works fine, both resolving the name with mDNS as well as with standard IP.
However, the HTTP GET does not work and the script hangs on it. Clearly the same script outside the container works just fine. This is the errors I am getting:
>>> requests.get("http://iot-ns-controller.local:8081")
Traceback (most recent call last):
File "/usr/lib/python3.10/site-packages/urllib3/connection.py", line 174, in _new_conn
conn = connection.create_connection(
File "/usr/lib/python3.10/site-packages/urllib3/util/connection.py", line 95, in create_connection
raise err
File "/usr/lib/python3.10/site-packages/urllib3/util/connection.py", line 85, in create_connection
sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3.10/site-packages/urllib3/connectionpool.py", line 703, in urlopen
httplib_response = self._make_request(
File "/usr/lib/python3.10/site-packages/urllib3/connectionpool.py", line 398, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/lib/python3.10/site-packages/urllib3/connection.py", line 239, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "/usr/lib/python3.10/http/client.py", line 1282, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.10/http/client.py", line 1328, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.10/http/client.py", line 1277, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.10/http/client.py", line 1037, in _send_output
self.send(msg)
File "/usr/lib/python3.10/http/client.py", line 975, in send
self.connect()
File "/usr/lib/python3.10/site-packages/urllib3/connection.py", line 205, in connect
conn = self._new_conn()
File "/usr/lib/python3.10/site-packages/urllib3/connection.py", line 186, in _new_conn
raise NewConnectionError(
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x7faa6658a0>: Failed to establish a new connection: [Errno 111] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3.10/site-packages/requests/adapters.py", line 489, in send
resp = conn.urlopen(
File "/usr/lib/python3.10/site-packages/urllib3/connectionpool.py", line 787, in urlopen
retries = retries.increment(
File "/usr/lib/python3.10/site-packages/urllib3/util/retry.py", line 592, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='iot-ns-controller.local', port=8081): Max retries exceeded with url: / (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7faa6658a0>: Failed to establish a new connection: [Errno 111] Connection refused'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/lib/python3.10/site-packages/requests/api.py", line 73, in get
return request("get", url, params=params, **kwargs)
File "/usr/lib/python3.10/site-packages/requests/api.py", line 59, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/lib/python3.10/site-packages/requests/sessions.py", line 587, in request
resp = self.send(prep, **send_kwargs)
File "/usr/lib/python3.10/site-packages/requests/sessions.py", line 701, in send
r = adapter.send(request, **kwargs)
File "/usr/lib/python3.10/site-packages/requests/adapters.py", line 565, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='iot-ns-controller.local', port=8081): Max retries exceeded with url: / (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7faa6658a0>: Failed to establish a new connection: [Errno 111] Connection refused'))
It also does not work if I query host.docker.internal and I get the same error

Related

requests.exceptions.ConnectionError: HTTPConnectionPool(host='localhost', port=5000): Max retries exceeded with url: /blockchain (Caused by NewConnect

enter image description herethis is my code where i get stuck
Traceback (most recent call last):
File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connection.py", line 159, in _new_conn
conn = connection.create_connection(
File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\util\connection.py", line 84, in create_connection
raise err
File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\util\connection.py", line 74, in create_connection
sock.connect(sa)
ConnectionRefusedError: [WinError 10061] No connection could be made because the target machine actively refused it
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connectionpool.py", line 670, in urlopen
httplib_response = self._make_request(
File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connectionpool.py", line 392, in _make_request
conn.request(method, url, **httplib_request_kw)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.752.0_x64__qbz5n2kfra8p0\lib\http\client.py", line 1282, in request
self._send_request(method, url, body, headers, encode_chunked)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.752.0_x64__qbz5n2kfra8p0\lib\http\client.py", line 1328, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.752.0_x64__qbz5n2kfra8p0\lib\http\client.py", line 1277, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.752.0_x64__qbz5n2kfra8p0\lib\http\client.py", line 1037, in _send_output
self.send(msg)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.752.0_x64__qbz5n2kfra8p0\lib\http\client.py", line 975, in send
self.connect()
File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connection.py", line 187, in connect
conn = self._new_conn()
File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connection.py", line 171, in _new_conn
raise NewConnectionError(
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x000001AD66840430>: Failed to establish a new connection: [WinError 10061] No connection could be made because the target machine actively refused it
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\adapters.py", line 439, in send
resp = conn.urlopen(
File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\connectionpool.py", line 726, in urlopen
retries = retries.increment(
File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\urllib3\util\retry.py", line 446, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='localhost', port=5000): Max retries exceeded with url: /blockchain (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x000001AD66840430>: Failed to establish a new connection: [WinError 10061] No connection could be made because the target machine actively refused it'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.752.0_x64__qbz5n2kfra8p0\lib\runpy.py", line 187, in _run_module_as_main
mod_name, mod_spec, code = _get_module_details(mod_name, _Error)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.752.0_x64__qbz5n2kfra8p0\lib\runpy.py", line 146, in _get_module_details
return _get_module_details(pkg_main_name, error)
File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.10_3.10.752.0_x64__qbz5n2kfra8p0\lib\runpy.py", line 110, in _get_module_details
__import__(pkg_name)
File "C:\Users\annun\python-blockchain\backend\app\__init__.py", line 82, in <module>
result = requests.get(f'http://localhost:{ROOT_PORT}/blockchain')
File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\api.py", line 75, in get
return request('get', url, params=params, **kwargs)
File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\api.py", line 60, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\sessions.py", line 533, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\sessions.py", line 646, in send
r = adapter.send(request, **kwargs)
File "C:\Users\annun\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\LocalCache\local-packages\Python310\site-packages\requests\adapters.py", line 516, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='localhost', port=5000): Max retries exceeded with url: /blockchain (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x000001AD66840430>: Failed to establish a new connection: [WinError 10061] No connection could be made because the target machine actively refused it'))
Post your code, with your question. Not the error report, we can't do anything with just the error report. It's not OUR code you're reporting on.
It does however look like you might've entered the wrong domain/IP to connect to, but since it's localhost you've probably misconfigured it.

I'm stuck in loading the data from xml file

import requests
try:
import xml.etree.cElementTree as et
except ImportError:
import xml.etree.ElementTree as et
user_key = authorized_key
doc_name = "F-C0032-001"
api_link = "http://opendata.cwb.gov.tw/opendataapi?dataid=%s&authorizationkey=%s" % (doc_name,user_key)
report = requests.get(url=api_link).text #The problem is here
Error:
Traceback (most recent call last):
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connection.py", line 169, in _new_conn
conn = connection.create_connection(
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\util\connection.py", line 96, in create_connection
raise err
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\util\connection.py", line 86, in create_connection
sock.connect(sa)
ConnectionRefusedError: [WinError 10061]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 394, in _make_request
conn.request(method, url, **httplib_request_kw)
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connection.py", line 234, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\http\client.py", line 1255, in request
self._send_request(method, url, body, headers, encode_chunked)
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\http\client.py", line 1301, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\http\client.py", line 1250, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\http\client.py", line 1010, in _send_output
self.send(msg)
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\http\client.py", line 950, in send
self.connect()
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connection.py", line 200, in connect
conn = self._new_conn()
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connection.py", line 181, in _new_conn
raise NewConnectionError(
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x000001F0DC29F940>: Failed to establish a new connection: [WinError 10061]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\requests\adapters.py", line 439, in send
resp = conn.urlopen(
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 755, in urlopen
retries = retries.increment(
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\util\retry.py", line 573, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='opendata.cwb.gov.tw', port=80): Max retries exceeded with url: /opendataapi?dataid=F-C0032-001&authorizationkey= (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x000001F0DC29F940>: Failed to establish a new connection: [WinError 10061]'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "D:/Python Books/Python/Ch17 Flask Web API/weatherdata.py", line 14, in <module>
report = requests.get(url=api_link,headers=headers).text
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\requests\api.py", line 76, in get
return request('get', url, params=params, **kwargs)
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\requests\api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\requests\sessions.py", line 542, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\requests\sessions.py", line 655, in send
r = adapter.send(request, **kwargs)
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\requests\adapters.py", line 516, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='opendata.cwb.gov.tw', port=80): Max retries exceeded with url: /opendataapi?dataid=F-C0032-001&authorizationkey= (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x000001F0DC29F940>: Failed to establish a new connection: [WinError 10061]'))
Actually my aim is to read the data and select the related location from the api_link but however the file is too bulky so is there any good ways or methods to properly read the related data from the xml file without any connection error messages? Since I do not understand what the error messages telling me how to deal with so if possible could anyone give some related solutions?
it looks like the error is not reading the xlm but when you send the request to the server, this error:
requests.exceptions.ConnectionError: HTTPConnectionPool(host='opendata.cwb.gov.tw', port=80): Max retries exceeded with url: /opendataapi?dataid=F-C0032-001&authorizationkey=
Is send when you're sending too many requests from same ip address in short period of time.
Have you tried this ?
https://stackoverflow.com/a/47475019/17044313
It should look like this:
import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
user_key = Authorization Key
doc_name = 'F-C0032-001'
api_link = 'http://opendata.cwb.gov.tw/opendataapi?dataid=%s&authorizationkey=%s'%(doc_name,user_key)
session = requests.Session()
retry = Retry(connect=3, backoff_factor=0.5)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
session.get(api_link)

Python request fails from google cloud platform (app engine & cloud shell)

I have a very simple python http request which works on my local machine, but does not work in google app engine or from the google cloud shell. There must be config that is messing with the request, but I can't seem to find the problem.
I am using python 3.7
The code that causes the exception is the following (I have replaced the actual URL by example.com)
(The service I need to call is listening on port 6123)
import requests
myrequest = requests.get("http://example.com:6123/something?param=123")
The full error message I get is the following:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 672, in urlopen
chunked=chunked,
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 387, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/local/lib/python3.7/http/client.py", line 1229, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/local/lib/python3.7/http/client.py", line 1275, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/local/lib/python3.7/http/client.py", line 1224, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/local/lib/python3.7/http/client.py", line 1016, in _send_output
self.send(msg)
File "/usr/local/lib/python3.7/http/client.py", line 956, in send
self.connect()
File "/usr/local/lib/python3.7/site-packages/urllib3/connection.py", line 184, in connect
conn = self._new_conn()
File "/usr/local/lib/python3.7/site-packages/urllib3/connection.py", line 169, in _new_conn
self, "Failed to establish a new connection: %s" % e
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x7f0b242cd710>: Failed to establish a new connection: [Errno -2] Name or service not known
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.7/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/usr/local/lib/python3.7/site-packages/urllib3/connectionpool.py", line 720, in urlopen
method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
File "/usr/local/lib/python3.7/site-packages/urllib3/util/retry.py", line 436, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='example.com', port=6123): Max retries exceeded with url: /something?param=123 (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f0b242cd710>: Failed to establish a new connection: [Errno -2] Name or service not known'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/local/lib/python3.7/site-packages/requests/api.py", line 75, in get
return request('get', url, params=params, **kwargs)
File "/usr/local/lib/python3.7/site-packages/requests/api.py", line 60, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/local/lib/python3.7/site-packages/requests/sessions.py", line 533, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python3.7/site-packages/requests/sessions.py", line 646, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python3.7/site-packages/requests/adapters.py", line 516, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='example.com', port=6123): Max retries exceeded with url: /something?param=123 (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f0b242cd710>: Failed to establish a new connection: [Errno -2] Name or service not known'))
Domain is not available from the outside, only from local network. Thanks Dustin Ingram!

list index out of range when listing all projects using python-gitlab

I'm trying to use python-gitlab to list all my gitlab projects. But when I use the following code to list part of my projects in gitlab, I got list index out of range error.
# file name: poc.py
import gitlab
gl = gitlab.Gitlab(
'http://my-gitlab.com:10080',
private_token='<my_private_token>'
)
projects = gl.projects.list(all=True)
for project in projects:
print(project.attributes['name'])
My gitlab server has more than 300 projects.
The following is the complete error message:
Traceback (most recent call last):
File "/Users/MyUserName/.pyenv/versions/3.6.5/lib/python3.6/site-packages/gitlab/__init__.py", line 783, in next
item = self._data[self._current]
IndexError: list index out of range
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/MyUserName/.pyenv/versions/3.6.5/lib/python3.6/site-packages/urllib3/connection.py", line 159, in _new_conn
(self._dns_host, self.port), self.timeout, **extra_kw)
File "/Users/MyUserName/.pyenv/versions/3.6.5/lib/python3.6/site-packages/urllib3/util/connection.py", line 80, in create_connection
raise err
File "/Users/MyUserName/.pyenv/versions/3.6.5/lib/python3.6/site-packages/urllib3/util/connection.py", line 70, in create_connection
sock.connect(sa)
ConnectionRefusedError: [Errno 61] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/MyUserName/.pyenv/versions/3.6.5/lib/python3.6/site-packages/urllib3/connectionpool.py", line 600, in urlopen
chunked=chunked)
File "/Users/MyUserName/.pyenv/versions/3.6.5/lib/python3.6/site-packages/urllib3/connectionpool.py", line 354, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/Users/MyUserName/.pyenv/versions/3.6.5/lib/python3.6/http/client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/Users/MyUserName/.pyenv/versions/3.6.5/lib/python3.6/http/client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/Users/MyUserName/.pyenv/versions/3.6.5/lib/python3.6/http/client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/Users/MyUserName/.pyenv/versions/3.6.5/lib/python3.6/http/client.py", line 1026, in _send_output
self.send(msg)
File "/Users/MyUserName/.pyenv/versions/3.6.5/lib/python3.6/http/client.py", line 964, in send
self.connect()
File "/Users/MyUserName/.pyenv/versions/3.6.5/lib/python3.6/site-packages/urllib3/connection.py", line 181, in connect
conn = self._new_conn()
File "/Users/MyUserName/.pyenv/versions/3.6.5/lib/python3.6/site-packages/urllib3/connection.py", line 168, in _new_conn
self, "Failed to establish a new connection: %s" % e)
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x107784be0>: Failed to establish a new connection: [Errno 61] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/MyUserName/.local/lib/python3.6/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/Users/MyUserName/.pyenv/versions/3.6.5/lib/python3.6/site-packages/urllib3/connectionpool.py", line 638, in urlopen
_stacktrace=sys.exc_info()[2])
File "/Users/MyUserName/.pyenv/versions/3.6.5/lib/python3.6/site-packages/urllib3/util/retry.py", line 398, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='my-gitlab.com', port=80): Max retries exceeded with url: /api/v4/projects?membership=false&order_by=created_at&owned=false&page=2&per_page=20&simple=false&sort=desc&starred=false&statistics=false&with_custom_attributes=false&with_issues_enabled=false&with_merge_requests_enabled=false (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x107784be0>: Failed to establish a new connection: [Errno 61] Connection refused',))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "poc.py", line 15, in <module>
projects = gl.projects.list(as_list=False, all=True)
File "/Users/MyUserName/.pyenv/versions/3.6.5/lib/python3.6/site-packages/gitlab/exceptions.py", line 255, in wrapped_f
return f(*args, **kwargs)
File "/Users/MyUserName/.pyenv/versions/3.6.5/lib/python3.6/site-packages/gitlab/mixins.py", line 133, in list
obj = self.gitlab.http_list(path, **data)
File "/Users/MyUserName/.pyenv/versions/3.6.5/lib/python3.6/site-packages/gitlab/__init__.py", line 597, in http_list
return list(GitlabList(self, url, query_data, **kwargs))
File "/Users/MyUserName/.pyenv/versions/3.6.5/lib/python3.6/site-packages/gitlab/__init__.py", line 779, in __next__
return self.next()
File "/Users/MyUserName/.pyenv/versions/3.6.5/lib/python3.6/site-packages/gitlab/__init__.py", line 788, in next
self._query(self._next_url)
File "/Users/MyUserName/.pyenv/versions/3.6.5/lib/python3.6/site-packages/gitlab/__init__.py", line 716, in _query
**kwargs)
File "/Users/MyUserName/.pyenv/versions/3.6.5/lib/python3.6/site-packages/gitlab/__init__.py", line 498, in http_request
result = self.session.send(prepped, timeout=timeout, **settings)
File "/Users/MyUserName/.local/lib/python3.6/site-packages/requests/sessions.py", line 646, in send
r = adapter.send(request, **kwargs)
File "/Users/MyUserName/.local/lib/python3.6/site-packages/requests/adapters.py", line 516, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='my-gitlab.com', port=80): Max retries exceeded with url: /api/v4/projects?membership=false&order_by=created_at&owned=false&page=2&per_page=20&simple=false&sort=desc&starred=false&statistics=false&with_custom_attributes=false&with_issues_enabled=false&with_merge_requests_enabled=false (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x107784be0>: Failed to establish a new connection: [Errno 61] Connection refused',))
The error message requests.exceptions.ConnectionError: HTTPConnectionPool(host='my-gitlab.com', port=80): Max retries exceeded says that it uses port 80 instead of 10080 to communicate with gitlab server. This is because there was an IndexError triggered at line 825 of gitlab/init.py. So python-gitlab tried to use port 80 to see if this port works.
I've checked that the length of self._data is 20. And I don't know why self._current was added to 20. self._data[20] triggers IndexError because the index's maximum value is 19.
For now, I use the following workaround to list all the projects:
import gitlab
gl = gitlab.Gitlab(
'http://my-gitlab.com:10080',
private_token='<my_private_token>'
)
current_page = 1
projects = dest_gitlab.projects.list(as_list=False)
total_pages = projects.total_pages
while (current_page <= total_pages):
projects = gl.projects.list(as_list=False, per_page=100, page=current_page)
for project in projects:
print(project.attributes['name'])
current_page += 1
But this is inconvenient to list all projects.
The document here doesn't mention anything about this error.
Anyone know how to list all projects using python-gitlab?
Is this a bug of python-gitlab?
Is len(self._data) == 20 an expected behavior when calling gl.projects.list(all=True)?
The reason for python-gitlab using port 80 to communicate with gitlab server is because this gitlab bug.
For now, I can only use the workaround or wait for gitlab to fix this bug.
import gitlab
gl = gitlab.Gitlab(
'http://my-gitlab.com:10080',
private_token='<my_private_token>'
)
current_page = 1
projects = dest_gitlab.projects.list(as_list=False)
total_pages = projects.total_pages
while (current_page <= total_pages):
projects = gl.projects.list(as_list=False, per_page=100, page=current_page)
for project in projects:
print(project.attributes['name'])
current_page += 1

Python automatic login with Requests and Beautifulsoup from list

trying to learn Python at the moment and am currently developing a small program which imports a list of login pages and tries your username/password automatically on them. Unfortunately I'm running into some errors and can't quite figure it out what the issue is. Please excuse the messy code by the way, still working on it.
Here's my source:
import requests
from bs4 import BeautifulSoup
import_file_path = input('Enter the path of the list to be tested: ')
export_file_path = input('Enter the path of where we should export the websites that worked to:: ')
with open(import_file_path, 'r') as panels:
panel_list = []
for line in panels:
panel_list.append(line)
x = 0
for panel in panel_list:
url = requests.get(panel)
soup = BeautifulSoup(url.content, "html.parser")
forms = soup.find_all("form")
action = soup.find('form').get('action')
values = {
soup.find_all("input")[0].get("name") : "user",
soup.find_all("input")[1].get("name") : "pass"
}
if "http://" or "https://" not in action:
action = 'http://' + action
r = requests.post(action, data=values)
print(r.content)
x += 1
Here are the errors I'm getting:
Enter the path of the website to be tested: list.txt
Enter the path of where we should export the vuln panels to: exit.txt
http://http://localhost/admin.php/vuln.php
Traceback (most recent call last):
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\urllib3\connection.py", line 141, in _new_conn
(self.host, self.port), self.timeout, **extra_kw)
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\urllib3\util\connection.py", line 60, in create_connection
for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\socket.py", line 743, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno 11001] getaddrinfo failed
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\urllib3\connectionpool.py", line 601, in urlopen
chunked=chunked)
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\urllib3\connectionpool.py", line 357, in _make_request
conn.request(method, url, **httplib_request_kw)
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 1239, in request
self._send_request(method, url, body, headers, encode_chunked)
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 1285, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 1234, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 1026, in _send_output
self.send(msg)
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\http\client.py", line 964, in send
self.connect()
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\urllib3\connection.py", line 166, in connect
conn = self._new_conn()
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\urllib3\connection.py", line 150, in _new_conn
self, "Failed to establish a new connection: %s" % e)
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x038B9070>: Failed to establish a new connection: [Errno 11001] getaddrinfo failed
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\requests\adapters.py", line 440, in send
timeout=timeout
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\urllib3\connectionpool.py", line 639, in urlopen
_stacktrace=sys.exc_info()[2])
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\urllib3\util\retry.py", line 388, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='http', port=80): Max retries exceeded with url: //localhost/admin.php/vuln.php (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x038B9070>: Failed to establish a new connection: [Errno 11001] getaddrinfo failed',))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "scan.py", line 35, in <module>
r = requests.post(action, data=values)
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\requests\api.py", line 112, in post
return request('post', url, data=data, json=json, **kwargs)
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\requests\api.py", line 58, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\requests\sessions.py", line 508, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\requests\sessions.py", line 618, in send
r = adapter.send(request, **kwargs)
File "C:\Users\User\AppData\Local\Programs\Python\Python36-32\lib\site-packages\requests\adapters.py", line 508, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='http', port=80): Max retries exceeded with url: //localhost/admin.php/vuln.php (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x038B9070>: Failed to establish a new connection: [Errno 11001] getaddrinfo failed',))

Categories

Resources