I've seen hundreds of posts with this same question but none of the answers ever work. I've tried doing everything on the post answers and they all don't work.
import requests
import random
def get_proxies():
request = requests.get('https://proxylist.geonode.com/api/proxy-list?limit=50&page=1&sort_by=lastChecked&sort_type=desc')
request = request.json()['data']
proxies = []
for proxy in request:
proxies.append(f'{proxy["ip"]}:{proxy["port"]}')
return proxies
def format_proxy(proxy):
return {'http': proxy, 'https': proxy}
if __name__ == '__main__':
proxies = get_proxies()
response =requests.get('https://api.ipify.org/', proxies=format_proxy(random.choice(proxies)), headers={"User-Agent": "Opera/9.80 (X11; Linux x86_64; U; de) Presto/2.2.15 Version/10.00"})
print(response.text)
I'm scraping the proxies from https://geonode.com/free-proxy-list/ and I keep getting the same error. I'm scraping new proxies every time I send a request so they shouldn't be invalid yet it's giving me a "max retry exceeded error"
File "/home/runner/proxies/venv/lib/python3.8/site-packages/urllib3/connectionpool.py", line 700, in urlopen
self._prepare_proxy(conn)
File "/home/runner/proxies/venv/lib/python3.8/site-packages/urllib3/connectionpool.py", line 994, in _prepare_proxy
conn.connect()
File "/home/runner/proxies/venv/lib/python3.8/site-packages/urllib3/connection.py", line 369, in connect
self._tunnel()
File "/nix/store/p21fdyxqb3yqflpim7g8s1mymgpnqiv7-python3-3.8.12/lib/python3.8/http/client.py", line 901, in _tunnel
(version, code, message) = response._read_status()
File "/nix/store/p21fdyxqb3yqflpim7g8s1mymgpnqiv7-python3-3.8.12/lib/python3.8/http/client.py", line 277, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/nix/store/p21fdyxqb3yqflpim7g8s1mymgpnqiv7-python3-3.8.12/lib/python3.8/socket.py", line 669, in readinto
return self._sock.recv_into(b)
ConnectionResetError: [Errno 104] Connection reset by peer
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/runner/proxies/venv/lib/python3.8/site-packages/requests/adapters.py", line 440, in send
resp = conn.urlopen(
File "/home/runner/proxies/venv/lib/python3.8/site-packages/urllib3/connectionpool.py", line 785, in urlopen
retries = retries.increment(
File "/home/runner/proxies/venv/lib/python3.8/site-packages/urllib3/util/retry.py", line 592, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='api.ipify.org', port=443): Max retries exceeded with url: / (Caused by ProxyError('Cannot connect to proxy.', ConnectionResetError(104, 'Connection reset by peer')))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "main.py", line 19, in <module>
response =requests.get('https://api.ipify.org/', proxies=proxy, headers={"User-Agent": "Opera/9.80 (X11; Linux x86_64; U; de) Presto/2.2.15 Version/10.00"})
File "/home/runner/proxies/venv/lib/python3.8/site-packages/requests/api.py", line 75, in get
return request('get', url, params=params, **kwargs)
File "/home/runner/proxies/venv/lib/python3.8/site-packages/requests/api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "/home/runner/proxies/venv/lib/python3.8/site-packages/requests/sessions.py", line 529, in request
resp = self.send(prep, **send_kwargs)
File "/home/runner/proxies/venv/lib/python3.8/site-packages/requests/sessions.py", line 645, in send
r = adapter.send(request, **kwargs)
File "/home/runner/proxies/venv/lib/python3.8/site-packages/requests/adapters.py", line 513, in send
raise ProxyError(e, request=request)
requests.exceptions.ProxyError: HTTPSConnectionPool(host='api.ipify.org', port=443): Max retries exceeded with url: / (Caused by ProxyError('Cannot connect to proxy.', ConnectionResetError(104, 'Connection reset by peer')))
Related
I need to parse a lot of pages and at some point I get an error:
requests.exceptions.SSLError: HTTPSConnectionPool(host='example.com', port=443): Max retries exceeded with url: /catalog/product-category/channels/videos/121 (Caused by SSLError(SSLEOFError(8
, 'EOF occurred in violation of protocol (_ssl.c:997)')))
I tried adding the following, but still the same error:
import urllib3
requests.packages.urllib3.disable_warnings()
requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS = 'ALL:#SECLEVEL=1'
my code example:
for page in range(1, pages):
response = requests.get(category + f"/{page}", headers={"User-Agent": ua.random}, verify=False)
soup = BeautifulSoup(response.text, "lxml")
...
log error:
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.10/threading.py", line 1016, in _bootstrap_inner
self.run()
File "/usr/local/lib/python3.10/threading.py", line 953, in run
self._target(*self._args, **self._kwargs)
File "/parser/main.py", line 44, in start
self.parse_urls()
File "/parser/main.py", line 35, in parse_urls
response = requests.get(category + f"/{page}", headers={"User-Agent": ua.random})
File "/usr/local/lib/python3.10/site-packages/requests/api.py", line 73, in get
return request("get", url, params=params, **kwargs)
File "/usr/local/lib/python3.10/site-packages/requests/api.py", line 59, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/local/lib/python3.10/site-packages/requests/sessions.py", line 587, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python3.10/site-packages/requests/sessions.py", line 701, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python3.10/site-packages/requests/adapters.py", line 563, in send
raise SSLError(e, request=request)
requests.exceptions.SSLError: HTTPSConnectionPool(host='example.com', port=443): Max retries exceeded with url: /catalog/product-category/channels/videos/116 (Caused by SSLError(SSLEOFError(8
, 'EOF occurred in violation of protocol (_ssl.c:997)')))
When sending a request with authentication, I get a requests.exceptions.SSLError error which you can See below.
proxies = { 'https' : "http://user:pass#ip:port/" }
url = "https://httpbin.org/ip"
numberResponse = requests.get(url,proxies=proxies).text
print(numberResponse)
The requests.exceptions.SSLError
Traceback (most recent call last):
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connectionpool.py", line 696, in urlopen
self._prepare_proxy(conn)
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connectionpool.py", line 964, in _prepare_proxy
conn.connect()
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connection.py", line 359, in connect
conn = self._connect_tls_proxy(hostname, conn)
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connection.py", line 496, in _connect_tls_proxy
return ssl_wrap_socket(
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\util\ssl_.py", line 428, in ssl_wrap_socket
ssl_sock = _ssl_wrap_socket_impl(
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\util\ssl_.py", line 472, in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\ssl.py", line 500, in wrap_socket
return self.sslsocket_class._create(
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\ssl.py", line 1040, in _create
self.do_handshake()
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\ssl.py", line 1309, in do_handshake
self._sslobj.do_handshake()
ssl.SSLEOFError: EOF occurred in violation of protocol (_ssl.c:1125)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\adapters.py", line 439, in send
resp = conn.urlopen(
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connectionpool.py", line 755, in urlopen
retries = retries.increment(
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\util\retry.py", line 573, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='httpbin.org', port=443): Max retries exceeded with url: /ip (Caused by SSLError(SSLEOFError(8, 'EOF occurred in violation of protocol (_ssl.c:1125)')))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "c:/Users/K_Yuk/OneDrive/Desktop/Gmail generator/test.py", line 15, in <module>
numberResponse = requests.get(url,proxies=proxies).text
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\api.py", line 76, in get
return request('get', url, params=params, **kwargs)
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\sessions.py", line 542, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\sessions.py", line 655, in send
r = adapter.send(request, **kwargs)
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\adapters.py", line 514, in send
raise SSLError(e, request=request)
requests.exceptions.SSLError: HTTPSConnectionPool(host='httpbin.org', port=443): Max retries exceeded with url: /ip (Caused by SSLError(SSLEOFError(8, 'EOF occurred in violation of protocol (_ssl.c:1125)')))
So then I tired verify=False as one of the requests.get() parameters but then get a requests.exceptions.ProxyError error which you can see below :
proxies = { 'https' : "http://user:pass#10.10.1.10:3128/"}
url = "https://httpbin.org/ip"
numberResponse = requests.get(url,proxies=proxies,verify=False).text
print(numberResponse)
The requests.exceptions.ProxyError
Traceback (most recent call last):
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connectionpool.py", line 696, in urlopen
self._prepare_proxy(conn)
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connectionpool.py", line 964, in _prepare_proxy
conn.connect()
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connection.py", line 359, in connect
conn = self._connect_tls_proxy(hostname, conn)
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connection.py", line 496, in _connect_tls_proxy
return ssl_wrap_socket(
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\util\ssl_.py", line 428, in ssl_wrap_socket
ssl_sock = _ssl_wrap_socket_impl(
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\util\ssl_.py", line 472, in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\ssl.py", line 500, in wrap_socket
return self.sslsocket_class._create(
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\ssl.py", line 1040, in _create
self.do_handshake()
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\ssl.py", line 1309, in do_handshake
self._sslobj.do_handshake()
FileNotFoundError: [Errno 2] No such file or directory
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\adapters.py", line 439, in send
resp = conn.urlopen(
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connectionpool.py", line 755, in urlopen
retries = retries.increment(
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\util\retry.py", line 573, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='httpbin.org', port=443): Max retries exceeded with url: /ip (Caused by ProxyError('Cannot connect to proxy.', FileNotFoundError(2, 'No such file or directory')))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "c:/Users/K_Yuk/OneDrive/Desktop/Gmail generator/test.py", line 15, in <module>
numberResponse = requests.get(url,proxies=proxies,verify=False).text
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\api.py", line 76, in get
return request('get', url, params=params, **kwargs)
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\sessions.py", line 542, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\sessions.py", line 655, in send
r = adapter.send(request, **kwargs)
File "C:\Users\K_Yuk\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\adapters.py", line 510, in send
raise ProxyError(e, request=request)
requests.exceptions.ProxyError: HTTPSConnectionPool(host='httpbin.org', port=443): Max retries exceeded with url: /ip (Caused by ProxyError('Cannot connect to proxy.', FileNotFoundError(2, 'No such file or directory')))
I tired to look every for the answer but nothing seems to work. I can't send a request with a proxy with
authentication. Any ideas?
The problem is very likely not the authentication. Unfortunately, you don't provide details of the proxy configuration and the URL you use for the proxy. The only thing you provide is:
proxies = { 'https' : eampleIpWithAuth }
Based on the reference to _connect_tls_proxy in the stacktrace the eampleIpWithAuth is very likely something like https://..., i.e. you try to access the proxy itself over HTTPS. Note that accessing a proxy over HTTPS is different from using a HTTP proxy for HTTPS. When accessing a HTTPS URL over a HTTPS proxy one essentially does double encryption to the proxy:
client --- [HTTPS wrapped inside HTTPS] --- proxy --- [HTTPS] --- server
Whereas with a HTTPS URL over a "normal" HTTP proxy there is only single encryption, i.e. it looks (simplified) like this:
client --- [HTTPS wrapped inside HTTP] --- proxy --- [HTTPS] --- server
Very likely the proxy you want to use is a plain HTTP proxy, and not a HTTPS proxy. This is actually the most common case.
The error happens since the proxy is not able to speak TLS but gets accessed by TLS. The fix is to use http://proxy and not https://proxy as the proxy address. Note that the latter worked in older versions of Python since proxy over HTTPS was not supported and a value of https:// for the protocol was treated the same as http://.
I had a similar problem with my script in python 3.9 version.
When I try to get some data from web server, I had exception like Your:
urllib3.exceptions.MaxRetryError
Simple add this line to configure Your requests
import requests
import urllib3
requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS = 'ALL:#SECLEVEL=1'
I hope, this help You to solve Your issue
I got error:
Traceback (most recent call last):
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\urllib3\connectionpool.py", line 696, in urlopen
self._prepare_proxy(conn)
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\urllib3\connectionpool.py", line 964, in _prepare_proxy
conn.connect()
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\urllib3\connection.py", line 364, in connect
conn = self._connect_tls_proxy(hostname, conn)
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\urllib3\connection.py", line 501, in _connect_tls_proxy
socket = ssl_wrap_socket(
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\urllib3\util\ssl_.py", line 453, in ssl_wrap_socket
ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls)
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\urllib3\util\ssl_.py", line 495, in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock)
File "C:\Users\user\AppData\Local\Programs\Python\Python310\lib\ssl.py", line 512, in wrap_socket
return self.sslsocket_class._create(
File "C:\Users\user\AppData\Local\Programs\Python\Python310\lib\ssl.py", line 1070, in _create
self.do_handshake()
File "C:\Users\user\AppData\Local\Programs\Python\Python310\lib\ssl.py", line 1341, in do_handshake
self._sslobj.do_handshake()
ssl.SSLEOFError: EOF occurred in violation of protocol (_ssl.c:997)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\requests\adapters.py", line 439, in send
resp = conn.urlopen(
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\urllib3\connectionpool.py", line 755, in urlopen
retries = retries.increment(
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\urllib3\util\retry.py", line 574, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='ipinfo.io', port=443): Max retries exceeded with url: /json (Caused by SSLError(SSLEOFError(8, 'EOF occurred in violation of protocol (_ssl.c:997)')))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\user\PycharmProjects\PythonLesson\text.py", line 7, in <module>
response = requests.get('https://ipinfo.io/json', proxies=proxies)
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\requests\api.py", line 75, in get
return request('get', url, params=params, **kwargs)
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\requests\api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\requests\sessions.py", line 542, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\requests\sessions.py", line 655, in send
r = adapter.send(request, **kwargs)
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\requests\adapters.py", line 514, in send
raise SSLError(e, request=request)
requests.exceptions.SSLError: HTTPSConnectionPool(host='ipinfo.io', port=443): Max retries exceeded with url: /json (Caused by SSLError(SSLEOFError(8, 'EOF occurred in violation of protocol (_ssl.c:997)')))
Process finished with exit code 1
Do I need to download any pip SSL? Or is it because of a proxy problem? Or add a pip to IDE like "Pycharm"?
I get every time error even when I change a proxy.
my code:
import requests
proxies = {
'https': 'https://150.129.148.99:35101'
}
response = requests.get('https://ipinfo.io/json', proxies=proxies)
print(response.json()['country'])
print(response.json()['region'])
This is a video https://www.youtube.com/watch?v=nnIye9pM22w in min 5:21 that I copied his code, and just for him it works and for me it does not
It's definitely a problem with proxy.
You can check it with curl like this:
curl -v -x https://150.129.148.99:35101 ipinfo.io
You'll have:
* Trying 150.129.148.99...
* TCP_NODELAY set
And nothing more. That means the proxy server doesn't respond correctly.
import requests
try:
import xml.etree.cElementTree as et
except ImportError:
import xml.etree.ElementTree as et
user_key = authorized_key
doc_name = "F-C0032-001"
api_link = "http://opendata.cwb.gov.tw/opendataapi?dataid=%s&authorizationkey=%s" % (doc_name,user_key)
report = requests.get(url=api_link).text #The problem is here
Error:
Traceback (most recent call last):
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connection.py", line 169, in _new_conn
conn = connection.create_connection(
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\util\connection.py", line 96, in create_connection
raise err
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\util\connection.py", line 86, in create_connection
sock.connect(sa)
ConnectionRefusedError: [WinError 10061]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 394, in _make_request
conn.request(method, url, **httplib_request_kw)
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connection.py", line 234, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\http\client.py", line 1255, in request
self._send_request(method, url, body, headers, encode_chunked)
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\http\client.py", line 1301, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\http\client.py", line 1250, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\http\client.py", line 1010, in _send_output
self.send(msg)
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\http\client.py", line 950, in send
self.connect()
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connection.py", line 200, in connect
conn = self._new_conn()
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connection.py", line 181, in _new_conn
raise NewConnectionError(
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x000001F0DC29F940>: Failed to establish a new connection: [WinError 10061]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\requests\adapters.py", line 439, in send
resp = conn.urlopen(
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 755, in urlopen
retries = retries.increment(
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\urllib3\util\retry.py", line 573, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='opendata.cwb.gov.tw', port=80): Max retries exceeded with url: /opendataapi?dataid=F-C0032-001&authorizationkey= (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x000001F0DC29F940>: Failed to establish a new connection: [WinError 10061]'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "D:/Python Books/Python/Ch17 Flask Web API/weatherdata.py", line 14, in <module>
report = requests.get(url=api_link,headers=headers).text
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\requests\api.py", line 76, in get
return request('get', url, params=params, **kwargs)
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\requests\api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\requests\sessions.py", line 542, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\requests\sessions.py", line 655, in send
r = adapter.send(request, **kwargs)
File "C:\Users\domin\AppData\Local\Programs\Python\Python39\lib\site-packages\requests\adapters.py", line 516, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='opendata.cwb.gov.tw', port=80): Max retries exceeded with url: /opendataapi?dataid=F-C0032-001&authorizationkey= (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x000001F0DC29F940>: Failed to establish a new connection: [WinError 10061]'))
Actually my aim is to read the data and select the related location from the api_link but however the file is too bulky so is there any good ways or methods to properly read the related data from the xml file without any connection error messages? Since I do not understand what the error messages telling me how to deal with so if possible could anyone give some related solutions?
it looks like the error is not reading the xlm but when you send the request to the server, this error:
requests.exceptions.ConnectionError: HTTPConnectionPool(host='opendata.cwb.gov.tw', port=80): Max retries exceeded with url: /opendataapi?dataid=F-C0032-001&authorizationkey=
Is send when you're sending too many requests from same ip address in short period of time.
Have you tried this ?
https://stackoverflow.com/a/47475019/17044313
It should look like this:
import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
user_key = Authorization Key
doc_name = 'F-C0032-001'
api_link = 'http://opendata.cwb.gov.tw/opendataapi?dataid=%s&authorizationkey=%s'%(doc_name,user_key)
session = requests.Session()
retry = Retry(connect=3, backoff_factor=0.5)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)
session.get(api_link)
I've been trying to access a website through a proxy in python using the requests module (I'm accessing api.ipify.org in my example because it should print the proxy's ip). I've tested the proxy using the same program before, and it has worked. But sometimes randomly when I run it, it gives me an error. Here is my code:
import requests
proxies = {
"http": "39.100.18.200:8080",
"https": "39.100.18.200:8080",
}
r = requests.get("http://api.ipify.org/", proxies=proxies)
print(r.text)
It should print: 39.100.18.200
And it has before, but randomly it will give me this error:
Traceback (most recent call last):
File "C:\Users\mmein\AppData\Local\Programs\Python\Python38-32\lib\site-packages\urllib3\connectionpool.py", line 665, in urlopen
httplib_response = self._make_request(
File "C:\Users\mmein\AppData\Local\Programs\Python\Python38-32\lib\site-packages\urllib3\connectionpool.py", line 421, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "C:\Users\mmein\AppData\Local\Programs\Python\Python38-32\lib\site-packages\urllib3\connectionpool.py", line 416, in _make_request
httplib_response = conn.getresponse()
File "C:\Users\mmein\AppData\Local\Programs\Python\Python38-32\lib\http\client.py", line 1322, in getresponse
response.begin()
File "C:\Users\mmein\AppData\Local\Programs\Python\Python38-32\lib\http\client.py", line 303, in begin
version, status, reason = self._read_status()
File "C:\Users\mmein\AppData\Local\Programs\Python\Python38-32\lib\http\client.py", line 272, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\mmein\AppData\Local\Programs\Python\Python38-32\lib\site-packages\requests\adapters.py", line 439, in send
resp = conn.urlopen(
File "C:\Users\mmein\AppData\Local\Programs\Python\Python38-32\lib\site-packages\urllib3\connectionpool.py", line 719, in urlopen
retries = retries.increment(
File "C:\Users\mmein\AppData\Local\Programs\Python\Python38-32\lib\site-packages\urllib3\util\retry.py", line 436, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='39.100.18.200', port=8080): Max retries exceeded with url: http://api.ipify.org/ (Caused by ProxyError('Cannot connect to proxy.', RemoteDisconnected('Remote end closed connection without response')))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\mmein\OneDrive\Desktop\pywork\nitrothruproxy.py", line 7, in <module>
r = requests.get("http://api.ipify.org/", proxies=proxies)
File "C:\Users\mmein\AppData\Local\Programs\Python\Python38-32\lib\site-packages\requests\api.py", line 76, in get
return request('get', url, params=params, **kwargs)
File "C:\Users\mmein\AppData\Local\Programs\Python\Python38-32\lib\site-packages\requests\api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Users\mmein\AppData\Local\Programs\Python\Python38-32\lib\site-packages\requests\sessions.py", line 530, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\mmein\AppData\Local\Programs\Python\Python38-32\lib\site-packages\requests\sessions.py", line 643, in send
r = adapter.send(request, **kwargs)
File "C:\Users\mmein\AppData\Local\Programs\Python\Python38-32\lib\site-packages\requests\adapters.py", line 510, in send
raise ProxyError(e, request=request)
requests.exceptions.ProxyError: HTTPConnectionPool(host='39.100.18.200', port=8080): Max retries exceeded with url: http://api.ipify.org/ (Caused by ProxyError('Cannot connect to proxy.', RemoteDisconnected('Remote end closed connection without response')))
[Finished in 2.4s with exit code 1]
[shell_cmd: python -u "C:\Users\mmein\OneDrive\Desktop\pywork\nitrothruproxy.py"]
[dir: C:\Users\mmein\OneDrive\Desktop\pywork]
[path: C:\Program Files (x86)\Common Files\Oracle\Java\javapath;C:\WINDOWS\system32;C:\WINDOWS;C:\WINDOWS\System32\Wbem;C:\WINDOWS\System32\WindowsPowerShell\v1.0\;C:\WINDOWS\System32\OpenSSH\;C:\Program Files\Git\cmd;C:\Program Files\nodejs\;C:\Users\mmein\AppData\Local\Programs\Python\Python38-32\Scripts\;C:\Users\mmein\AppData\Local\Programs\Python\Python38-32\;C:\Users\mmein\AppData\Local\Programs\Python\Python38\Scripts\;C:\Users\mmein\AppData\Local\Programs\Python\Python38\;C:\Program Files\MySQL\MySQL Shell 8.0\bin\;C:\Users\mmein\AppData\Local\Microsoft\WindowsApps;C:\Users\mmein\AppData\Local\Programs\Microsoft VS Code\bin;C:\Users\mmein\AppData\Local\atom\bin;C:\Users\mmein\OneDrive\Desktop\gecko\geckodriver.exe;C:\ffmpeg\bin;C:\MinGW\bin;C:\Program Files\heroku\bin;C:\Users\mmein\AppData\Roaming\npm;%USERPROFILE%\AppData\Local\Microsoft\WindowsApps;]