I need to access a website using selenium and Chrome using python.
Below is shortened version of my code.
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.service import Service
# PROXY='https://myproxy.com:3128'
PROXY = 'https://username:password#myproxy.com:3128'
proxyuser='username' #this is the proxy user name used in above string
proxypwd='password' #this is the proxy password used in above string
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=%s' % PROXY)
chrome_options.add_argument("ignore-certificate-errors");
chrome = webdriver.Chrome(options=chrome_options,service=Service(ChromeDriverManager().install()))
chrome.get("https://www.google.com")
while True:
print('ok')
I am behind a corporate proxy server which requires authentication.
I am not sure how to pass login credentials and proxy settings for Install of chromedriver
When above code is run without proxy, it works as expected.
But it gives error as follows when run using proxy connection:
[WDM] - ====== WebDriver manager ======
[WDM] - Current google-chrome version is 105.0.5195
[WDM] - Get LATEST chromedriver version for 105.0.5195 google-chrome
Traceback (most recent call last):
File "D:\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 703, in urlopen
httplib_response = self._make_request(
File "D:\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 386, in _make_request
self._validate_conn(conn)
File "D:\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 1040, in _validate_conn
conn.connect()
File "D:\Python\Python39\lib\site-packages\urllib3\connection.py", line 414, in connect
self.sock = ssl_wrap_socket(
File "D:\Python\Python39\lib\site-packages\urllib3\util\ssl_.py", line 449, in ssl_wrap_socket
ssl_sock = _ssl_wrap_socket_impl(
File "D:\Python\Python39\lib\site-packages\urllib3\util\ssl_.py", line 493, in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
File "D:\Python\Python39\lib\ssl.py", line 500, in wrap_socket
return self.sslsocket_class._create(
File "D:\Python\Python39\lib\ssl.py", line 1040, in _create
self.do_handshake()
File "D:\Python\Python39\lib\ssl.py", line 1309, in do_handshake
self._sslobj.do_handshake()
ConnectionResetError: [WinError 10054] An existing connection was forcibly closed by the remote host
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "D:\Python\Python39\lib\site-packages\requests\adapters.py", line 489, in send
resp = conn.urlopen(
File "D:\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 785, in urlopen
retries = retries.increment(
File "D:\Python\Python39\lib\site-packages\urllib3\util\retry.py", line 550, in increment
raise six.reraise(type(error), error, _stacktrace)
File "D:\Python\Python39\lib\site-packages\urllib3\packages\six.py", line 769, in reraise
raise value.with_traceback(tb)
File "D:\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 703, in urlopen
httplib_response = self._make_request(
File "D:\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 386, in _make_request
self._validate_conn(conn)
File "D:\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 1040, in _validate_conn
conn.connect()
File "D:\Python\Python39\lib\site-packages\urllib3\connection.py", line 414, in connect
self.sock = ssl_wrap_socket(
File "D:\Python\Python39\lib\site-packages\urllib3\util\ssl_.py", line 449, in ssl_wrap_socket
ssl_sock = _ssl_wrap_socket_impl(
File "D:\Python\Python39\lib\site-packages\urllib3\util\ssl_.py", line 493, in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
File "D:\Python\Python39\lib\ssl.py", line 500, in wrap_socket
return self.sslsocket_class._create(
File "D:\Python\Python39\lib\ssl.py", line 1040, in _create
self.do_handshake()
File "D:\Python\Python39\lib\ssl.py", line 1309, in do_handshake
self._sslobj.do_handshake()
urllib3.exceptions.ProtocolError: ('Connection aborted.', ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "D:\eclipse-workspace\Essential Training\test.py", line 17, in <module>
chrome = webdriver.Chrome(options=chrome_options,service=Service(ChromeDriverManager().install()))
File "D:\Python\Python39\lib\site-packages\webdriver_manager\chrome.py", line 37, in install
driver_path = self._get_driver_path(self.driver)
File "D:\Python\Python39\lib\site-packages\webdriver_manager\core\manager.py", line 29, in _get_driver_path
binary_path = self.driver_cache.find_driver(driver)
File "D:\Python\Python39\lib\site-packages\webdriver_manager\core\driver_cache.py", line 95, in find_driver
driver_version = driver.get_version()
File "D:\Python\Python39\lib\site-packages\webdriver_manager\core\driver.py", line 42, in get_version
self.get_latest_release_version()
File "D:\Python\Python39\lib\site-packages\webdriver_manager\drivers\chrome.py", line 44, in get_latest_release_version
resp = self._http_client.get(url=latest_release_url)
File "D:\Python\Python39\lib\site-packages\webdriver_manager\core\http.py", line 31, in get
resp = requests.get(url=url, verify=self._ssl_verify, **kwargs)
File "D:\Python\Python39\lib\site-packages\requests\api.py", line 73, in get
return request("get", url, params=params, **kwargs)
File "D:\Python\Python39\lib\site-packages\requests\api.py", line 59, in request
return session.request(method=method, url=url, **kwargs)
File "D:\Python\Python39\lib\site-packages\requests\sessions.py", line 587, in request
resp = self.send(prep, **send_kwargs)
File "D:\Python\Python39\lib\site-packages\requests\sessions.py", line 701, in send
r = adapter.send(request, **kwargs)
File "D:\Python\Python39\lib\site-packages\requests\adapters.py", line 547, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', ConnectionResetError(10054, 'An existing connection was forcibly closed by the remote host', None, 10054, None))
Bypassing proxy is not possible because of corporate policies
request("get", url, params=params, **kwargs)
this line in errors i believe is the issue.
it seems the driver is not able to determine the new version available on net as the request function here should have proxy settings - i am not able to see the **kwargs values, ideally - i believe, **kwargs should have proxy argument. - i am not sure but.
I have modified the code as follows.
essentially i have now used custom download manager
and this is worling
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from webdriver_manager.core.download_manager import WDMDownloadManager
from webdriver_manager.core.http import HttpClient
from selenium.webdriver.chrome.service import Service
from requests import Response
import urllib3
import requests
import os
os.environ['WDM_SSL_VERIFY'] = '0'
capabilities = webdriver.DesiredCapabilities.CHROME
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
PROXY = "https://username:password#myproxy.com:3128"
proxyuser='username'
proxypwd='password'
opt = webdriver.ChromeOptions()
opt.add_argument('--proxy-server=%s' % PROXY)
opt.add_argument("ignore-certificate-errors")
class CustomHttpClient(HttpClient):
def get(self, url, params=None) -> Response:
proxies={'http': 'http://username:password#myproxy.com:3128',
'https': 'http://username:password#myproxy.com:3128',
}
return requests.get(url, params,proxies=proxies,verify=False)
http_client = CustomHttpClient()
download_manager = WDMDownloadManager(http_client)
chrome = webdriver.Chrome(service=Service(ChromeDriverManager(download_manager=download_manager).install()),options=opt)
chrome.get("https://www.google.com")
while True:
print('ok')
Related
I am trying to run a very simple selenium python code in PyCharm, but it throws out the following error every time when I call driver = webdriver.Firefox(driver_path).
/home/kimilao/PycharmProjects/pythonOne/main.py:7: DeprecationWarning: firefox_profile has been deprecated, please pass in an Options object
driver = webdriver.Firefox(driver_path)
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 169, in _new_conn
conn = connection.create_connection(
File "/usr/lib/python3/dist-packages/urllib3/util/connection.py", line 96, in create_connection
raise err
File "/usr/lib/python3/dist-packages/urllib3/util/connection.py", line 86, in create_connection
sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 394, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 234, in request
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
File "/usr/lib/python3.10/http/client.py", line 1282, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/lib/python3.10/http/client.py", line 1328, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/lib/python3.10/http/client.py", line 1277, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/lib/python3.10/http/client.py", line 1037, in _send_output
self.send(msg)
File "/usr/lib/python3.10/http/client.py", line 975, in send
self.connect()
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 200, in connect
conn = self._new_conn()
File "/usr/lib/python3/dist-packages/urllib3/connection.py", line 181, in _new_conn
raise NewConnectionError(
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x7fedb7be9c00>: Failed to establish a new connection: [Errno 111] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/kimilao/PycharmProjects/pythonOne/main.py", line 7, in <module>
driver = webdriver.Firefox(driver_path)
File "/home/kimilao/.local/lib/python3.10/site-packages/selenium/webdriver/firefox/webdriver.py", line 177, in __init__
super().__init__(
File "/home/kimilao/.local/lib/python3.10/site-packages/selenium/webdriver/remote/webdriver.py", line 277, in __init__
self.start_session(capabilities, browser_profile)
File "/home/kimilao/.local/lib/python3.10/site-packages/selenium/webdriver/remote/webdriver.py", line 370, in start_session
response = self.execute(Command.NEW_SESSION, parameters)
File "/home/kimilao/.local/lib/python3.10/site-packages/selenium/webdriver/remote/webdriver.py", line 433, in execute
response = self.command_executor.execute(driver_command, params)
File "/home/kimilao/.local/lib/python3.10/site-packages/selenium/webdriver/remote/remote_connection.py", line 344, in execute
return self._request(command_info[0], url, body=data)
File "/home/kimilao/.local/lib/python3.10/site-packages/selenium/webdriver/remote/remote_connection.py", line 366, in _request
response = self._conn.request(method, url, body=body, headers=headers)
File "/usr/lib/python3/dist-packages/urllib3/request.py", line 78, in request
return self.request_encode_body(
File "/usr/lib/python3/dist-packages/urllib3/request.py", line 170, in request_encode_body
return self.urlopen(method, url, **extra_kw)
File "/usr/lib/python3/dist-packages/urllib3/poolmanager.py", line 375, in urlopen
response = conn.urlopen(method, u.request_uri, **kw)
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 783, in urlopen
return self.urlopen(
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 783, in urlopen
return self.urlopen(
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 783, in urlopen
return self.urlopen(
File "/usr/lib/python3/dist-packages/urllib3/connectionpool.py", line 755, in urlopen
retries = retries.increment(
File "/usr/lib/python3/dist-packages/urllib3/util/retry.py", line 574, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='localhost', port=57917): Max retries exceeded with url: /session (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7fedb7be9c00>: Failed to establish a new connection: [Errno 111] Connection refused'))
Here's my full code:
#!/usr/bin/python3
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
driver_path = "/home/kimilao/Downloads/geckodriver-v0.31.0-linux64"
driver = webdriver.Firefox(driver_path)
driver.get("https://www.python.org/")
print(driver.title)
driver.quit()
The problem is that I can run this code without the errors when using Sublime Text, vscode and terminal, but not PyCharm, and I wonder why this happens to me. Thanks!
Python: 3.10.4
Selenium: 4.3.0
Firefox: 102.0 (64-bit)
Gecko Driver: geckodriver-v0.31.0-linux64
I'm pretty sure this happens because PyCharm automatically creates and enters a virtual environment, where you probably didn't run a pip install. To make sure, try launching these commands from vscode:
source venv/bin/activate
python your_script.py
deactivate # after executing your script, to exit the venv
If you do get the same error, then you didn't set up a venv properly
i use Pycharm (albeit on windows) and my import statements and driver object are completely different. Maybe you can give it a try and see if it works for you?
from selenium import webdriver
from selenium.webdriver.firefox.service import Service
from webdriver_manager.firefox import GeckoDriverManager
driver = webdriver.Firefox(service=Service(GeckoDriverManager().install()))
Note: this is under the assumption that the driver as been downloaded and installed (also, confirm your internet connection. I experienced something similar using chrome, but my problem was the internet connection)
Getting an Error for this Basic Appium Script while attempting to run it on a real device. Everything works fine if I used the Appium Inspector, but once I try to run the same code in PyCharm im getting an Error. Below is the JSON format, as Im pretty new to this library I don't know if I've done it correctly or not.
from appium import webdriver
desired_cap = {
"deviceName": "R9AN60B4CCJ",
"platformName": "Android",
"app": "C:\\Users\\John Doe\\AppData\\Local\\Android\\Sdk\\platform-tools\\airmirror2.apk"
} #The Capabilities to install an app from the apk file on your Computer
driver = webdriver.Remote('https://localhost:4723/wd/hub', desired_cap)
#Similar to Selenium; Declaring the Driver. The Appium Server should already be started
the Error is below
Traceback (most recent call last):
File "C:\Users\John Doe\PycharmProjects\SelTest\venv\lib\site-packages\urllib3\connectionpool.py", line 600, in urlopen
chunked=chunked)
File "C:\Users\John Doe\PycharmProjects\SelTest\venv\lib\site-packages\urllib3\connectionpool.py", line 343, in _make_request
self._validate_conn(conn)
File "C:\Users\John Doe\PycharmProjects\SelTest\venv\lib\site-packages\urllib3\connectionpool.py", line 849, in _validate_conn
conn.connect()
File "C:\Users\John Doe\PycharmProjects\SelTest\venv\lib\site-packages\urllib3\connection.py", line 356, in connect
ssl_context=context)
File "C:\Users\John Doe\PycharmProjects\SelTest\venv\lib\site-packages\urllib3\util\ssl_.py", line 359, in ssl_wrap_socket
return context.wrap_socket(sock, server_hostname=server_hostname)
File "C:\Users\John Doe\Miniconda3\lib\ssl.py", line 423, in wrap_socket
session=session
File "C:\Users\John Doe\Miniconda3\lib\ssl.py", line 870, in _create
self.do_handshake()
File "C:\Users\John Doe\Miniconda3\lib\ssl.py", line 1139, in do_handshake
self._sslobj.do_handshake()
ssl.SSLError: [SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:1076)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:/Users/John Doe/PycharmProjects/SelTest/venv/FirstTest.py", line 11, in <module>
driver = webdriver.Remote('https://localhost:4723/wd/hub', desired_cap)
File "C:\Users\John Doe\PycharmProjects\SelTest\venv\lib\site-packages\appium\webdriver\webdriver.py", line 275, in __init__
AppiumConnection(command_executor, keep_alive=keep_alive), desired_capabilities, browser_profile, proxy
File "C:\Users\John Doe\PycharmProjects\SelTest\venv\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 269, in __init__
self.start_session(capabilities, browser_profile)
File "C:\Users\John Doe\PycharmProjects\SelTest\venv\lib\site-packages\appium\webdriver\webdriver.py", line 369, in start_session
response = self.execute(RemoteCommand.NEW_SESSION, parameters)
File "C:\Users\John Doe\PycharmProjects\SelTest\venv\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 423, in execute
response = self.command_executor.execute(driver_command, params)
File "C:\Users\John Doe\PycharmProjects\SelTest\venv\lib\site-packages\selenium\webdriver\remote\remote_connection.py", line 333, in execute
return self._request(command_info[0], url, body=data)
File "C:\Users\John Doe\PycharmProjects\SelTest\venv\lib\site-packages\selenium\webdriver\remote\remote_connection.py", line 355, in _request
resp = self._conn.request(method, url, body=body, headers=headers)
File "C:\Users\John Doe\PycharmProjects\SelTest\venv\lib\site-packages\urllib3\request.py", line 72, in request
**urlopen_kw)
File "C:\Users\John Doe\PycharmProjects\SelTest\venv\lib\site-packages\urllib3\request.py", line 150, in request_encode_body
return self.urlopen(method, url, **extra_kw)
File "C:\Users\John Doe\PycharmProjects\SelTest\venv\lib\site-packages\urllib3\poolmanager.py", line 322, in urlopen
response = conn.urlopen(method, u.request_uri, **kw)
File "C:\Users\John Doe\PycharmProjects\SelTest\venv\lib\site-packages\urllib3\connectionpool.py", line 667, in urlopen
**response_kw)
File "C:\Users\John Doe\PycharmProjects\SelTest\venv\lib\site-packages\urllib3\connectionpool.py", line 667, in urlopen
**response_kw)
File "C:\Users\John Doe\PycharmProjects\SelTest\venv\lib\site-packages\urllib3\connectionpool.py", line 667, in urlopen
**response_kw)
File "C:\Users\John Doe\PycharmProjects\SelTest\venv\lib\site-packages\urllib3\connectionpool.py", line 638, in urlopen
_stacktrace=sys.exc_info()[2])
File "C:\Users\John Doe\PycharmProjects\SelTest\venv\lib\site-packages\urllib3\util\retry.py", line 398, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='localhost', port=4723): Max retries exceeded with url: /wd/hub/session (Caused by SSLError(SSLError(1, '[SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:1076)')))
I dont know if I need a specific version of urllib3 in this venv.
I am attempting to send a request to Binance api (crypto exchange) for the latest price of a specific symbol of crypto every ten seconds. However after between 5 to 20 hours it stops working throwing this error:
Traceback (most recent call last): File
"C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\site-packages\urllib3\connectionpool.py",
line 703, in urlopen
httplib_response = self._make_request( File "C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\site-packages\urllib3\connectionpool.py",
line 386, in _make_request
self._validate_conn(conn) File "C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\site-packages\urllib3\connectionpool.py",
line 1040, in validate_conn
conn.connect() File "C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\site-packages\urllib3\connection.py",
line 416, in connect
self.sock = ssl_wrap_socket( File "C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\site-packages\urllib3\util\ssl.py",
line 449, in ssl_wrap_socket
ssl_sock = ssl_wrap_socket_impl( File "C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\site-packages\urllib3\util\ssl.py",
line 493, in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock, server_hostname=server_hostname) File
"C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\ssl.py",
line 512, in wrap_socket
return self.sslsocket_class._create( File "C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\ssl.py",
line 1070, in _create
self.do_handshake() File "C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\ssl.py",
line 1341, in do_handshake
self._sslobj.do_handshake() ConnectionResetError: [WinError 10054] An existing connection was forcibly closed by the remote host
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File
"C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\site-packages\requests\adapters.py",
line 440, in send
resp = conn.urlopen( File "C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\site-packages\urllib3\connectionpool.py",
line 785, in urlopen
retries = retries.increment( File "C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\site-packages\urllib3\util\retry.py",
line 550, in increment
raise six.reraise(type(error), error, _stacktrace) File "C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\site-packages\urllib3\packages\six.py",
line 769, in reraise
raise value.with_traceback(tb) File "C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\site-packages\urllib3\connectionpool.py",
line 703, in urlopen
httplib_response = self._make_request( File "C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\site-packages\urllib3\connectionpool.py",
line 386, in _make_request
self._validate_conn(conn) File "C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\site-packages\urllib3\connectionpool.py",
line 1040, in validate_conn
conn.connect() File "C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\site-packages\urllib3\connection.py",
line 416, in connect
self.sock = ssl_wrap_socket( File "C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\site-packages\urllib3\util\ssl.py",
line 449, in ssl_wrap_socket
ssl_sock = ssl_wrap_socket_impl( File "C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\site-packages\urllib3\util\ssl.py",
line 493, in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock, server_hostname=server_hostname) File
"C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\ssl.py",
line 512, in wrap_socket
return self.sslsocket_class._create( File "C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\ssl.py",
line 1070, in _create
self.do_handshake() File "C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\ssl.py",
line 1341, in do_handshake
self._sslobj.do_handshake() urllib3.exceptions.ProtocolError: ('Connection aborted.', ConnectionResetError(10054, 'An existing
connection was forcibly closed by the remote host', None, 10054,
None))
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File
"c:\Users\nrspa\Desktop\CryptoBot v2.6 (24h)\main (market).py", line
113, in
CB.main() File "c:\Users\nrspa\Desktop\CryptoBot v2.6 (24h)\main (market).py", line 68, in main
self.array = self.create_array() File "c:\Users\nrspa\Desktop\CryptoBot v2.6 (24h)\main (market).py", line
46, in create_array
self.array.append(self.get_price()) File "c:\Users\nrspa\Desktop\CryptoBot v2.6 (24h)\main (market).py", line
22, in get_price
response = requests.get(url, headers = {"User-Agent" : "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like
Gecko) Chrome/51.0.2704.103 Safari/537.36"}) File
"C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\site-packages\requests\api.py",
line 75, in get
return request('get', url, params=params, **kwargs) File "C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\site-packages\requests\api.py",
line 61, in request
return session.request(method=method, url=url, **kwargs) File "C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\site-packages\requests\sessions.py",
line 529, in request
resp = self.send(prep, **send_kwargs) File "C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\site-packages\requests\sessions.py",
line 645, in send
r = adapter.send(request, **kwargs) File "C:\Users\nrspa\AppData\Local\Programs\Python\Python310\lib\site-packages\requests\adapters.py",
line 501, in send
raise ConnectionError(err, request=request) requests.exceptions.ConnectionError: ('Connection aborted.',
ConnectionResetError(10054, 'An existing connection was forcibly
closed by the remote host', None, 10054, None))
Also note that this is my code (it is part of a class) it gets the price from Binance every 10 seconds until the desired array length is reached, at which point it processes the data:
import json
import requests
import time
def get_price(self):
url = f"https://api.binance.com/api/v3/ticker/price?symbol={self.symbol}"
response = requests.get(url, headers = {"User-Agent" : "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"})
json_response = json.loads(response.text)
return float(json_response['price'])
def create_array(self):
i = len(self.array)
if i < self.array_len:
while i < self.array_len:
time.sleep(self.time_interval)
self.array.append(self.get_price())
i = i + 1
return self.array
else:
main()
The issue can be caused by a "disagreement" between both sides. It can be that the connection timed out or the server closed it.
The fix is to catch this exception and open the connection again.
Python requests not working for a website API but works in Browser (chrome). it used to work earlier but from the past 2 months its not working. the api i am using is yts.mx api.
I'll attach the code snippet below:
import requests
a = requests.get('https://yts.mx/api/v2/list_movies.json')
This is the error i am getting:
Traceback (most recent call last):
File "D:\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 670, in urlopen
httplib_response = self._make_request(
File "D:\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 426, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "D:\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 421, in _make_request
httplib_response = conn.getresponse()
File "D:\Programs\Python\Python39\lib\http\client.py", line 1347, in getresponse
response.begin()
File "D:\Programs\Python\Python39\lib\http\client.py", line 307, in begin
version, status, reason = self._read_status()
File "D:\Programs\Python\Python39\lib\http\client.py", line 268, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "D:\Programs\Python\Python39\lib\socket.py", line 704, in readinto
return self._sock.recv_into(b)
File "D:\Programs\Python\Python39\lib\ssl.py", line 1241, in recv_into
return self.read(nbytes, buffer)
File "D:\Programs\Python\Python39\lib\ssl.py", line 1099, in read
return self._sslobj.read(len, buffer)
TimeoutError: [WinError 10060] A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "D:\Programs\Python\Python39\lib\site-packages\requests\adapters.py", line 439, in send
resp = conn.urlopen(
File "D:\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 726, in urlopen
retries = retries.increment(
File "D:\Programs\Python\Python39\lib\site-packages\urllib3\util\retry.py", line 410, in increment
raise six.reraise(type(error), error, _stacktrace)
File "D:\Programs\Python\Python39\lib\site-packages\urllib3\packages\six.py", line 734, in reraise
raise value.with_traceback(tb)
File "D:\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 670, in urlopen
httplib_response = self._make_request(
File "D:\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 426, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "D:\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 421, in _make_request
httplib_response = conn.getresponse()
File "D:\Programs\Python\Python39\lib\http\client.py", line 1347, in getresponse
response.begin()
File "D:\Programs\Python\Python39\lib\http\client.py", line 307, in begin
version, status, reason = self._read_status()
File "D:\Programs\Python\Python39\lib\http\client.py", line 268, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "D:\Programs\Python\Python39\lib\socket.py", line 704, in readinto
return self._sock.recv_into(b)
File "D:\Programs\Python\Python39\lib\ssl.py", line 1241, in recv_into
return self.read(nbytes, buffer)
File "D:\Programs\Python\Python39\lib\ssl.py", line 1099, in read
return self._sslobj.read(len, buffer)
urllib3.exceptions.ProtocolError: ('Connection aborted.', TimeoutError(10060, 'A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond', None, 10060, None))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "D:\Programs\Python\Python39\test.py", line 8, in <module>
a = r.get('https://yts.mx/api/v2/list_movies.json')
File "D:\Programs\Python\Python39\lib\site-packages\requests\api.py", line 76, in get
return request('get', url, params=params, **kwargs)
File "D:\Programs\Python\Python39\lib\site-packages\requests\api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "D:\Programs\Python\Python39\lib\site-packages\requests\sessions.py", line 530, in request
resp = self.send(prep, **send_kwargs)
File "D:\Programs\Python\Python39\lib\site-packages\requests\sessions.py", line 643, in send
r = adapter.send(request, **kwargs)
File "D:\Programs\Python\Python39\lib\site-packages\requests\adapters.py", line 498, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', TimeoutError(10060, 'A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond', None, 10060, None))
and sometimes getting error like forcibly closed due to existing connection, this is the trace:
Traceback (most recent call last):
File "D:\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 670, in urlopen
httplib_response = self._make_request(
File "D:\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 381, in _make_request
self._validate_conn(conn)
File "D:\Programs\Python\Python39\lib\site-packages\urllib3\connectionpool.py", line 978, in _validate_conn
conn.connect()
File "D:\Programs\Python\Python39\lib\site-packages\urllib3\connection.py", line 362, in connect
self.sock = ssl_wrap_socket(
File "D:\Programs\Python\Python39\lib\site-packages\urllib3\util\ssl_.py", line 386, in ssl_wrap_socket
return context.wrap_socket(sock, server_hostname=server_hostname)
File "D:\Programs\Python\Python39\lib\ssl.py", line 500, in wrap_socket
return self.sslsocket_class._create(
File "D:\Programs\Python\Python39\lib\ssl.py", line 1040, in _create
self.do_handshake()
File "D:\Programs\Python\Python39\lib\ssl.py", line 1309, in do_handshake
self._sslobj.do_handshake()
ConnectionResetError: [WinError 10054] An existing connection was forcibly closed by the remote host
This used to perfectly work a month ago or so. Tried various methods seen on web.
UPDATE
Turns out the issue is with the IP . When i use a VPN connection it works perfectly. However i dont need a VPN to access it from my web browser. any Idea why this is happening?
is there a way to bypass this?
Since you mentioned, that the browser request goes through but the python request does not, it crossed my mind, that the web site iteself blocks non-browser-requests. I found this question here, that dealt with that problem. The solution was to add a custom header to the python request to make it look like a browser request:
import requests
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.76 Safari/537.36'} # This is chrome, you can set whatever browser you like
url = 'https://yts.mx/api/v2/list_movies.json'
a = requests.get(url,headers)
print(a.content)
With your request, you then send the cutom header as the second parameter in your get-request. I tried the script above and it worked for me, but also your original script worked on my connection.
I just finished automate the boring stuff with python, now i'm just practicing.
I try running the code below but it fails . Other sites has no issue connecting .
Searched for other questions they say add user:agent, it works for amazon.com but not for this site .
I tried allow_redirects = False and ,verify=False too , they are not working
I am using Windows 10 , Python 3.8.5 , requests 2.24.0
my code ;
import requests
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'}
print('1')
res = requests.get('https://freida.ama-assn.org/search/list?spec=42931',headers=headers)
print('2')
print(res.status_code)
res.raise_for_status()
This code does not prints '2' fails at requests.get
Error code as follows :
C:\Users\User198\PycharmProjects\pythonProject\venv\Scripts\python.exe C:/Users/User198/PycharmProjects/pythonProject/amaResidency.py
1
Traceback (most recent call last):
File "C:\Users\User198\PycharmProjects\pythonProject\venv\lib\site-packages\urllib3\connectionpool.py", line 670, in urlopen
httplib_response = self._make_request(
File "C:\Users\User198\PycharmProjects\pythonProject\venv\lib\site-packages\urllib3\connectionpool.py", line 381, in _make_request
self._validate_conn(conn)
File "C:\Users\User198\PycharmProjects\pythonProject\venv\lib\site-packages\urllib3\connectionpool.py", line 978, in _validate_conn
conn.connect()
File "C:\Users\User198\PycharmProjects\pythonProject\venv\lib\site-packages\urllib3\connection.py", line 362, in connect
self.sock = ssl_wrap_socket(
File "C:\Users\User198\PycharmProjects\pythonProject\venv\lib\site-packages\urllib3\util\ssl_.py", line 384, in ssl_wrap_socket
return context.wrap_socket(sock, server_hostname=server_hostname)
File "C:\Users\User198\AppData\Local\Programs\Python\Python38-32\lib\ssl.py", line 500, in wrap_socket
return self.sslsocket_class._create(
File "C:\Users\User198\AppData\Local\Programs\Python\Python38-32\lib\ssl.py", line 1040, in _create
self.do_handshake()
File "C:\Users\User198\AppData\Local\Programs\Python\Python38-32\lib\ssl.py", line 1309, in do_handshake
self._sslobj.do_handshake()
OSError: [Errno 0] Error
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\User198\PycharmProjects\pythonProject\venv\lib\site-packages\requests\adapters.py", line 439, in send
resp = conn.urlopen(
File "C:\Users\User198\PycharmProjects\pythonProject\venv\lib\site-packages\urllib3\connectionpool.py", line 726, in urlopen
retries = retries.increment(
File "C:\Users\User198\PycharmProjects\pythonProject\venv\lib\site-packages\urllib3\util\retry.py", line 403, in increment
raise six.reraise(type(error), error, _stacktrace)
File "C:\Users\User198\PycharmProjects\pythonProject\venv\lib\site-packages\urllib3\packages\six.py", line 734, in reraise
raise value.with_traceback(tb)
File "C:\Users\User198\PycharmProjects\pythonProject\venv\lib\site-packages\urllib3\connectionpool.py", line 670, in urlopen
httplib_response = self._make_request(
File "C:\Users\User198\PycharmProjects\pythonProject\venv\lib\site-packages\urllib3\connectionpool.py", line 381, in _make_request
self._validate_conn(conn)
File "C:\Users\User198\PycharmProjects\pythonProject\venv\lib\site-packages\urllib3\connectionpool.py", line 978, in _validate_conn
conn.connect()
File "C:\Users\User198\PycharmProjects\pythonProject\venv\lib\site-packages\urllib3\connection.py", line 362, in connect
self.sock = ssl_wrap_socket(
File "C:\Users\User198\PycharmProjects\pythonProject\venv\lib\site-packages\urllib3\util\ssl_.py", line 384, in ssl_wrap_socket
return context.wrap_socket(sock, server_hostname=server_hostname)
File "C:\Users\User198\AppData\Local\Programs\Python\Python38-32\lib\ssl.py", line 500, in wrap_socket
return self.sslsocket_class._create(
File "C:\Users\User198\AppData\Local\Programs\Python\Python38-32\lib\ssl.py", line 1040, in _create
self.do_handshake()
File "C:\Users\User198\AppData\Local\Programs\Python\Python38-32\lib\ssl.py", line 1309, in do_handshake
self._sslobj.do_handshake()
urllib3.exceptions.ProtocolError: ('Connection aborted.', OSError(0, 'Error'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:/Users/User198/PycharmProjects/pythonProject/amaResidency.py", line 6, in <module>
print(res.status_code)
File "C:\Users\User198\PycharmProjects\pythonProject\venv\lib\site-packages\requests\api.py", line 76, in get
return request('get', url, params=params, **kwargs)
File "C:\Users\User198\PycharmProjects\pythonProject\venv\lib\site-packages\requests\api.py", line 61, in request
return session.request(method=method, url=url, **kwargs)
File "C:\Users\User198\PycharmProjects\pythonProject\venv\lib\site-packages\requests\sessions.py", line 530, in request
resp = self.send(prep, **send_kwargs)
File "C:\Users\User198\PycharmProjects\pythonProject\venv\lib\site-packages\requests\sessions.py", line 643, in send
r = adapter.send(request, **kwargs)
File "C:\Users\User198\PycharmProjects\pythonProject\venv\lib\site-packages\requests\adapters.py", line 498, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', OSError(0, 'Error'))
So, seems like i fixed the issue .
In my case code was working via my cellphone internet but Wi-Fi connection was failing for this particular site .
I unchecked "Microsoft LLDP Protocol Driver" from Control panel => network and internet => network connections => LeftClick your connection and properties => Networking tab => Uncheck "Microsoft LLDP Protocol Driver"
Everything was checked except "microsoft network adapter multiplexor protocol"
Now "Microsoft LLDP Protocol Driver" is not checked too.
This solution worked for me
You may try ignoring SSL certificate verification using verify = False. It drastically reduced my count of similar ConnectionErrors.
res = requests.get('https://freida.ama-assn.org/search/list?spec=42931', headers=headers, verify=False)
Note that when verify is set to False, requests will accept any TLS certificate presented by the server, and will ignore hostname mismatches and/or expired certificates, which will make your application vulnerable to man-in-the-middle (MitM) attacks.