I need to parse a lot of pages and at some point I get an error:
requests.exceptions.SSLError: HTTPSConnectionPool(host='example.com', port=443): Max retries exceeded with url: /catalog/product-category/channels/videos/121 (Caused by SSLError(SSLEOFError(8
, 'EOF occurred in violation of protocol (_ssl.c:997)')))
I tried adding the following, but still the same error:
import urllib3
requests.packages.urllib3.disable_warnings()
requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS = 'ALL:#SECLEVEL=1'
my code example:
for page in range(1, pages):
response = requests.get(category + f"/{page}", headers={"User-Agent": ua.random}, verify=False)
soup = BeautifulSoup(response.text, "lxml")
...
log error:
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.10/threading.py", line 1016, in _bootstrap_inner
self.run()
File "/usr/local/lib/python3.10/threading.py", line 953, in run
self._target(*self._args, **self._kwargs)
File "/parser/main.py", line 44, in start
self.parse_urls()
File "/parser/main.py", line 35, in parse_urls
response = requests.get(category + f"/{page}", headers={"User-Agent": ua.random})
File "/usr/local/lib/python3.10/site-packages/requests/api.py", line 73, in get
return request("get", url, params=params, **kwargs)
File "/usr/local/lib/python3.10/site-packages/requests/api.py", line 59, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/local/lib/python3.10/site-packages/requests/sessions.py", line 587, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python3.10/site-packages/requests/sessions.py", line 701, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python3.10/site-packages/requests/adapters.py", line 563, in send
raise SSLError(e, request=request)
requests.exceptions.SSLError: HTTPSConnectionPool(host='example.com', port=443): Max retries exceeded with url: /catalog/product-category/channels/videos/116 (Caused by SSLError(SSLEOFError(8
, 'EOF occurred in violation of protocol (_ssl.c:997)')))
Related
I've seen hundreds of posts with this same question but none of the answers ever work. I've tried doing everything on the post answers and they all don't work.
import requests
import random
def get_proxies():
request = requests.get('https://proxylist.geonode.com/api/proxy-list?limit=50&page=1&sort_by=lastChecked&sort_type=desc')
request = request.json()['data']
proxies = []
for proxy in request:
proxies.append(f'{proxy["ip"]}:{proxy["port"]}')
return proxies
def format_proxy(proxy):
return {'http': proxy, 'https': proxy}
if __name__ == '__main__':
proxies = get_proxies()
response =requests.get('https://api.ipify.org/', proxies=format_proxy(random.choice(proxies)), headers={"User-Agent": "Opera/9.80 (X11; Linux x86_64; U; de) Presto/2.2.15 Version/10.00"})
print(response.text)
I'm scraping the proxies from https://geonode.com/free-proxy-list/ and I keep getting the same error. I'm scraping new proxies every time I send a request so they shouldn't be invalid yet it's giving me a "max retry exceeded error"
File "/home/runner/proxies/venv/lib/python3.8/site-packages/urllib3/connectionpool.py", line 700, in urlopen
self._prepare_proxy(conn)
File "/home/runner/proxies/venv/lib/python3.8/site-packages/urllib3/connectionpool.py", line 994, in _prepare_proxy
conn.connect()
File "/home/runner/proxies/venv/lib/python3.8/site-packages/urllib3/connection.py", line 369, in connect
self._tunnel()
File "/nix/store/p21fdyxqb3yqflpim7g8s1mymgpnqiv7-python3-3.8.12/lib/python3.8/http/client.py", line 901, in _tunnel
(version, code, message) = response._read_status()
File "/nix/store/p21fdyxqb3yqflpim7g8s1mymgpnqiv7-python3-3.8.12/lib/python3.8/http/client.py", line 277, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/nix/store/p21fdyxqb3yqflpim7g8s1mymgpnqiv7-python3-3.8.12/lib/python3.8/socket.py", line 669, in readinto
return self._sock.recv_into(b)
ConnectionResetError: [Errno 104] Connection reset by peer
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/runner/proxies/venv/lib/python3.8/site-packages/requests/adapters.py", line 440, in send
resp = conn.urlopen(
File "/home/runner/proxies/venv/lib/python3.8/site-packages/urllib3/connectionpool.py", line 785, in urlopen
retries = retries.increment(
File "/home/runner/proxies/venv/lib/python3.8/site-packages/urllib3/util/retry.py", line 592, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='api.ipify.org', port=443): Max retries exceeded with url: / (Caused by ProxyError('Cannot connect to proxy.', ConnectionResetError(104, 'Connection reset by peer')))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "main.py", line 19, in <module>
response =requests.get('https://api.ipify.org/', proxies=proxy, headers={"User-Agent": "Opera/9.80 (X11; Linux x86_64; U; de) Presto/2.2.15 Version/10.00"})
File "/home/runner/proxies/venv/lib/python3.8/site-packages/requests/api.py", line 75, in get
return request('get', url, params=params, **kwargs)
File "/home/runner/proxies/venv/lib/python3.8/site-packages/requests/api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "/home/runner/proxies/venv/lib/python3.8/site-packages/requests/sessions.py", line 529, in request
resp = self.send(prep, **send_kwargs)
File "/home/runner/proxies/venv/lib/python3.8/site-packages/requests/sessions.py", line 645, in send
r = adapter.send(request, **kwargs)
File "/home/runner/proxies/venv/lib/python3.8/site-packages/requests/adapters.py", line 513, in send
raise ProxyError(e, request=request)
requests.exceptions.ProxyError: HTTPSConnectionPool(host='api.ipify.org', port=443): Max retries exceeded with url: / (Caused by ProxyError('Cannot connect to proxy.', ConnectionResetError(104, 'Connection reset by peer')))
I got error:
Traceback (most recent call last):
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\urllib3\connectionpool.py", line 696, in urlopen
self._prepare_proxy(conn)
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\urllib3\connectionpool.py", line 964, in _prepare_proxy
conn.connect()
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\urllib3\connection.py", line 364, in connect
conn = self._connect_tls_proxy(hostname, conn)
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\urllib3\connection.py", line 501, in _connect_tls_proxy
socket = ssl_wrap_socket(
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\urllib3\util\ssl_.py", line 453, in ssl_wrap_socket
ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls)
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\urllib3\util\ssl_.py", line 495, in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock)
File "C:\Users\user\AppData\Local\Programs\Python\Python310\lib\ssl.py", line 512, in wrap_socket
return self.sslsocket_class._create(
File "C:\Users\user\AppData\Local\Programs\Python\Python310\lib\ssl.py", line 1070, in _create
self.do_handshake()
File "C:\Users\user\AppData\Local\Programs\Python\Python310\lib\ssl.py", line 1341, in do_handshake
self._sslobj.do_handshake()
ssl.SSLEOFError: EOF occurred in violation of protocol (_ssl.c:997)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\requests\adapters.py", line 439, in send
resp = conn.urlopen(
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\urllib3\connectionpool.py", line 755, in urlopen
retries = retries.increment(
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\urllib3\util\retry.py", line 574, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='ipinfo.io', port=443): Max retries exceeded with url: /json (Caused by SSLError(SSLEOFError(8, 'EOF occurred in violation of protocol (_ssl.c:997)')))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\user\PycharmProjects\PythonLesson\text.py", line 7, in <module>
response = requests.get('https://ipinfo.io/json', proxies=proxies)
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\requests\api.py", line 75, in get
return request('get', url, params=params, **kwargs)
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\requests\api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\requests\sessions.py", line 542, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\requests\sessions.py", line 655, in send
r = adapter.send(request, **kwargs)
File "C:\Users\user\PycharmProjects\PythonLesson\venv\lib\site-packages\requests\adapters.py", line 514, in send
raise SSLError(e, request=request)
requests.exceptions.SSLError: HTTPSConnectionPool(host='ipinfo.io', port=443): Max retries exceeded with url: /json (Caused by SSLError(SSLEOFError(8, 'EOF occurred in violation of protocol (_ssl.c:997)')))
Process finished with exit code 1
Do I need to download any pip SSL? Or is it because of a proxy problem? Or add a pip to IDE like "Pycharm"?
I get every time error even when I change a proxy.
my code:
import requests
proxies = {
'https': 'https://150.129.148.99:35101'
}
response = requests.get('https://ipinfo.io/json', proxies=proxies)
print(response.json()['country'])
print(response.json()['region'])
This is a video https://www.youtube.com/watch?v=nnIye9pM22w in min 5:21 that I copied his code, and just for him it works and for me it does not
It's definitely a problem with proxy.
You can check it with curl like this:
curl -v -x https://150.129.148.99:35101 ipinfo.io
You'll have:
* Trying 150.129.148.99...
* TCP_NODELAY set
And nothing more. That means the proxy server doesn't respond correctly.
i tried to make a web site traffic increaser but i got an error like "Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x000001B6E7CC5930>,"
So how will i fix that? i have a problem about proxys timeout.
and thats the Traceback over here :
Traceback (most recent call last):
File "C:\Users\emreb\AppData\Local\Programs\Python\Python310\Web Traffic\proxychecker.py", line 182, in <module>
sending()
File "C:\Users\emreb\AppData\Local\Programs\Python\Python310\Web Traffic\proxychecker.py", line 176, in sending
requests.get(url, proxies=proxies2, verify=False, timeout=5)
File "C:\Users\emreb\AppData\Local\Programs\Python\Python310\lib\site-packages\requests\api.py", line 75, in get
return request('get', url, params=params, **kwargs)
File "C:\Users\emreb\AppData\Local\Programs\Python\Python310\lib\site-packages\requests\api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Users\emreb\AppData\Local\Programs\Python\Python310\lib\site-packages\requests\sessions.py", line 542, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\emreb\AppData\Local\Programs\Python\Python310\lib\site-packages\requests\sessions.py", line 655, in send
r = adapter.send(request, **kwargs)
File "C:\Users\emreb\AppData\Local\Programs\Python\Python310\lib\site-packages\requests\adapters.py", line 504, in send
raise ConnectTimeout(e, request=request)
requests.exceptions.ConnectTimeout: HTTPSConnectionPool(host='grabify.link', port=443): Max retries exceeded with url: /4VFAZU (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x000001B6E7CC5930>, 'Connection to 91.230.199.174 timed out. (connect timeout=5)'))
maybe you can fix this problem by setting timeout parameter. like this
session.get(url, headers=header, proxy=myproxy,timeout=30)
whenever i run this code its shows me the requests.exceptions.ConnectionError: HTTPSConnectionPool error
import requests
def download(url):
get_response = requests.get(url)
file_name =url.split("/")[-1]
print(file_name)
with open("sample.txt", "wb") as out:
out.write(get_response.content)
download("https://i.pinimg.com/originals/ca/76/0b/ca760b70976b52578da88e06973af542.jpg")
And this is the error
Traceback (most recent call last):
File "download_file.py", line 10, in <module>
download("https://i.pinimg.com/originals/ca/76/0b/ca760b70976b52578da88e06973af542.jpg")
File "download_file.py", line 4, in download
get_response = requests.get(url)
File "/usr/local/lib/python2.7/dist-packages/requests/api.py", line 76, in get
return request('get', url, params=params, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/requests/api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 530, in request
resp = self.send(prep, **send_kwargs)
File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 643, in send
r = adapter.send(request, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/requests/adapters.py", line 516, in send
raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPSConnectionPool(host='i.pinimg.com', port=443): Max retries exceeded with url: /originals/ca/76/0b/ca760b70976b52578da88e06973af542.jpg (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7f6bebc83350>: Failed to establish a new connection: [Errno -3] Temporary failure in name resolution',))
This is a internet problem check your wifi, and make sure the URL you wrote is correct.
import pandas_datareader.data as web
import datetime
start = datetime.datetime(2016, 2, 19)
end = datetime.datetime(2016, 2, 22)
gs = web.DataReader("078930.KS", "yahoo", start, end)
it's a just simple code and it gives this kind of error
C:\ProgramData\Anaconda3\python.exe
C:/Users/asdf/PycharmProjects/untitled1/asdf.py Traceback (most recent
call last): File
"C:\ProgramData\Anaconda3\lib\site-packages\urllib3\connectionpool.py",
line 588, in urlopen
conn = self._get_conn(timeout=pool_timeout) File "C:\ProgramData\Anaconda3\lib\site-packages\urllib3\connectionpool.py",
line 248, in _get_conn
return conn or self._new_conn() File "C:\ProgramData\Anaconda3\lib\site-packages\urllib3\connectionpool.py",
line 816, in _new_conn
raise SSLError("Can't connect to HTTPS URL because the SSL " urllib3.exceptions.SSLError: Can't connect to HTTPS URL because the
SSL module is not available.
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File
"C:\ProgramData\Anaconda3\lib\site-packages\requests\adapters.py",
line 449, in send
timeout=timeout File "C:\ProgramData\Anaconda3\lib\site-packages\urllib3\connectionpool.py",
line 638, in urlopen
_stacktrace=sys.exc_info()[2]) File "C:\ProgramData\Anaconda3\lib\site-packages\urllib3\util\retry.py",
line 398, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause)) urllib3.exceptions.MaxRetryError:
HTTPSConnectionPool(host='finance.yahoo.com', port=443): Max retries
exceeded with url:
/quote/078930.KS/history?period1=1455822000&period2=1456167599&interval=1d&frequency=1d&filter=history (Caused by SSLError("Can't connect to HTTPS URL because the SSL module
is not available."))
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File
"C:/Users/asdf/PycharmProjects/untitled1/asdf.py", line 5, in
gs = web.DataReader("078930.KS", "yahoo", start, end) File "C:\ProgramData\Anaconda3\lib\site-packages\pandas_datareader\data.py",
line 310, in DataReader
session=session).read() File "C:\ProgramData\Anaconda3\lib\site-packages\pandas_datareader\base.py",
line 210, in read
params=self._get_params(self.symbols)) File "C:\ProgramData\Anaconda3\lib\site-packages\pandas_datareader\yahoo\daily.py",
line 129, in _read_one_data
resp = self._get_response(url, params=params) File "C:\ProgramData\Anaconda3\lib\site-packages\pandas_datareader\base.py",
line 132, in _get_response
headers=headers) File "C:\ProgramData\Anaconda3\lib\site-packages\requests\sessions.py",
line 546, in get
return self.request('GET', url, **kwargs) File "C:\ProgramData\Anaconda3\lib\site-packages\requests\sessions.py",
line 533, in request
resp = self.send(prep, **send_kwargs) File "C:\ProgramData\Anaconda3\lib\site-packages\requests\sessions.py",
line 646, in send
r = adapter.send(request, **kwargs) File "C:\ProgramData\Anaconda3\lib\site-packages\requests\adapters.py",
line 514, in send
raise SSLError(e, request=request) requests.exceptions.SSLError: HTTPSConnectionPool(host='finance.yahoo.com', port=443): Max retries
exceeded with url:
/quote/078930.KS/history?period1=1455822000&period2=1456167599&interval=1d&frequency=1d&filter=history (Caused by SSLError("Can't connect to HTTPS URL because the SSL module
is not available."))
Process finished with exit code 1
I'm using 32bit anaconda version 3.7.3
Any Idea...what's wrong?