I try to connect http://username:password#ip:port proxy and options.add_argument(f'--proxy-server=%s' % proxy) not working. Can you help me?..
To connect a proxy through ChromeDriver you can use the following code block:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
PROXY = "ip:port"
options = Options()
options.add_argument('--proxy-server=%s' % PROXY)
driver = webdriver.Chrome(options=options, executable_path=r'C:\path\to\chromedriver.exe')
Related
I'm trying to use a Proxy in selenium python, but when i test the code on a web page that show my ip the proxy doesn't change my ip adress.
Here there is the code
import undetected_chromedriver as uc
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
option = Options()
driver = uc.Chrome(options = option)
PROXY = "139.162.214.146"
option.add_argument('--proxy-server=%s' % PROXY)
option.add_argument('--disable-extensions')
option.add_argument('--disable-notifications')
option.add_argument('--no-sandbox')
option.add_argument('--disable-dev-shm-usage')
option.add_argument('--disable-gpu')
option.add_argument('--mute-audio')
option.add_argument('--ignore-certificate-errors')
option.add_argument('--disable-logging')
driver.get("https://www.whatismyip.com/")
input ()
The code should change mi ip but this doesn't appen, i tried to change the proxy but the problem is the same.
I can see multiple issues here. Try using the following code:
import undetected_chromedriver as uc
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
option = Options()
PROXY = "139.162.214.146"
option.add_argument('--proxy-server=%s' % PROXY)
option.add_argument('--disable-extensions')
option.add_argument('--disable-notifications')
option.add_argument('--no-sandbox')
option.add_argument('--disable-dev-shm-usage')
option.add_argument('--disable-gpu')
option.add_argument('--mute-audio')
option.add_argument('--ignore-certificate-errors')
option.add_argument('--disable-logging')
driver = uc.Chrome(options = option)
driver.get("https://www.whatismyip.com/")
input ()
Also note, that you need to specify a port for your proxy as following: PROXY = "139.162.214.146:proxy_port"
I was trying to get around the request limit of GitHub contribution graph (e.g.,https://github.com/crobby/webhook/graphs/contributors) in my web-scraping) during web scraping. So I decide to use webdriver on Tor.
I can open my web driver with the Tor browser. But it cannot stuck at the connecting stage as shown in the screenshot.
I can open links with the web driver, but I still encountered the request limit after it scraped several links. Does anyone have a hint about the potential issue?
Here is my code:
from selenium import webdriver
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
from selenium.webdriver.firefox.options import Options
import os
torexe = os.popen(r'C:/Users/fredr/Desktop/Tor Browser/Browser/TorBrowser/Tor/tor.exe')
profile = FirefoxProfile(r'C:/Users/fredr/Desktop/Tor Browser/Browser/TorBrowser/Data/Browser/profile.default')
profile.set_preference('network.proxy.type', 1)
profile.set_preference('network.proxy.socks', '127.0.0.1')
profile.set_preference('network.proxy.socks_port', 9050)
profile.set_preference("network.proxy.socks_remote_dns", False)
profile.update_preferences()
options = Options()
options.binary_location = r'C:/Users/fredr/Desktop/Tor Browser/Browser/firefox.exe'
driver = webdriver.Firefox(firefox_profile= profile, executable_path=r'C:/Users/fredr/Downloads/geckodriver.exe', options=options)
driver.get("http://check.torproject.org")
How is it possible to change the proxy server in Selenium after starting the driver?
I saw several threads on this topic, but none of the answers were correct.
You can use not only Chrome but also Firefox.
If you have any ideas on how to change the proxy when the driver is open, please write.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.proxy import Proxy, ProxyType
from selenium.webdriver.support.ui import WebDriverWait
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
Path = ChromeDriverManager().install()
options = webdriver.ChromeOptions()
options.add_argument('--no-sandbox')
options.add_argument('--disable-setuid-sandbox')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--window-size=600,400')
options.add_argument('--ignore-certificate-errors')
options.add_argument('--disable-accelerated-2d-canvas')
# options.add_argument('--no-zygote')
# options.add_argument('--single-process')
options.add_argument('--disable-gpu')
options.add_argument('--headless')
proxy_file = open("proxy.txt", "r", encoding="utf-8", errors="ignore").readlines()
proxy = ((random.choice(proxy_file)).replace("\n", ""))
options.add_argument('--proxy-server=%s' % proxy)
browser = webdriver.Chrome(Path, options=options)
browser.get('https://google.com')
# Here the proxy should change
browser.get('https://google.com')
No, you won't be able to change the proxy server using Selenium after starting the driver and the Browsing Context.
When you configure an instance of a ChromeDriver with ChromeOptions() to span a new Chrome Browsing Context the configuration gets baked within the chromedriver executable which will persist for the lifetime of the WebDriver and being uneditable. So you can't modify/add any existing/new configuration through ChromeOptions() class to the WebDriver instance which is currently in execution.
Even if you are able to extract the ChromeDriver and ChromeSession attributes e.g. Session ID, Cookies, UserAgent and other session attributes from the already initiated ChromeDriver and Chrome Browsing Session still you won't be able to change the set of attributes of the ChromeDriver.
A cleaner way would be to quit() the existing ChromeDriver and Chrome Browser instances gracefully and then span a new set of ChromeDriver and Chrome Browser instance with the new set of proxy configuration as follows:
options = webdriver.ChromeOptions()
options.add_argument('--no-sandbox')
options.add_argument('--disable-setuid-sandbox')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--window-size=600,400')
options.add_argument('--ignore-certificate-errors')
options.add_argument('--disable-accelerated-2d-canvas')
options.add_argument('--disable-gpu')
options.add_argument('--headless')
urls_to_visit = ['https://www.google.com/', 'https://stackoverflow.com/']
proxies = open("proxy.txt", "r", encoding="utf-8", errors="ignore").readlines()
for i in range(0, len(urls_to_visit)):
proxy = ((random.choice(proxies)).replace("\n", ""))
options.add_argument('--proxy-server=%s' % proxy)
browser = webdriver.Chrome(Path, options=options)
browser.get("{}".format(urls_to_visit[i]))
# perform the tasks
driver.quit()
References
You can find a couple of relevant discussions in:
How to rotate Selenium webrowser IP address
am trying to use selenium to change my IP using a code I found
but am getting an error:
selenium.common.exceptions.WebDriverException: Message: 'chromedriver' executable needs to be in PATH
`from selenium import webdriver
PROXY = "23.23.23.23:3128" # IP:PORT or HOST:PORT
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=http://%s' % PROXY)
chrome = webdriver.Chrome(chrome_options=chrome_options)
chrome.get("http://whatismyipaddress.com")`
Here, I hope it helps :)
from selenium import webdriver
from selenium.webdriver.common.proxy import Proxy, ProxyType
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = "ip_addr:port"
prox.socks_proxy = "ip_addr:port"
prox.ssl_proxy = "ip_addr:port"
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)
driver = webdriver.Chrome(desired_capabilities=capabilities)
I solved this error by adding the executable path to the chrome variable
Here is my code :
from selenium import webdriver
from selenium.webdriver.common.proxy import Proxy, ProxyType
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=ipaddress:port')
driver = webdriver.Chrome(executable_path="your executable path",chrome_options=chrome_options)
driver.get('https://www.myip.com/')
How can I use proxy server using selenium and google chrome? I attached the code and I'm not sure if this will change the actual proxy server.
# selenium imports
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import random
PROXY ="88.157.149.250:8080";
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=%s' % PROXY)
# //a[starts-with(#href, 'https://www.amazon.com/')]/#href
LINKS_XPATH = '//*[contains(#id,"result")]/div/div[3]/div[1]/a'
browser = webdriver.Chrome(executable_path="C:\\Users\Andrei\Downloads\chromedriver_win32\chromedriver.exe",
chrome_options=chrome_options)
browser.get(
'https://www.amazon.com/s/ref=lp_11444071011_nr_p_8_1/132-3636705-4291947?rh=n%3A3375251%2Cn%3A%213375301%2Cn%3A10971181011%2Cn%3A11444071011%2Cp_8%3A2229059011')
links = browser.find_elements_by_xpath(LINKS_XPATH)
for link in links:
href = link.get_attribute('href')
print(href)
from selenium import webdriver
PROXY = "88.157.149.250:8080" # IP:PORT or HOST:PORT
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=%s' % PROXY)
chrome = webdriver.Chrome(chrome_options=chrome_options)
chrome.get("http://google.com")
You can open the page https://www.whatismyip.com/my-ip-information/
chrome.get("https://www.whatismyip.com/my-ip-information/")
proxy = 192.168.22.1:8080
if proxy != None:
print('\nProxy ativado: ',proxy)
#chrome_options.add_argument('--proxy-server=%s' % proxy)
webdriver.DesiredCapabilities.CHROME['proxy'] = {
"httpProxy": proxy,
"ftpProxy": proxy,
"sslProxy": proxy,
"proxyType": "MANUAL",
}
webdriver.DesiredCapabilities.CHROME['acceptSslCerts']=True
print(webdriver.DesiredCapabilities.CHROME)