I try to connect http://username:password#ip:port proxy and options.add_argument(f'--proxy-server=%s' % proxy) not working. Can you help me?..
To connect a proxy through ChromeDriver you can use the following code block:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
PROXY = "ip:port"
options = Options()
options.add_argument('--proxy-server=%s' % PROXY)
driver = webdriver.Chrome(options=options, executable_path=r'C:\path\to\chromedriver.exe')
I'm trying to set proxy on selenium-webdriver.
I tried different methods, I think there are some changes for setting proxy in chrome headless.
can anyone help me?
these are the methods I tried.
First
from selenium.webdriver.common.proxy import Proxy, ProxyType
options = webdriver.ChromeOptions()
proxy = Proxy()
proxy.proxyType = ProxyType.MANUAL
proxy.autodetect = False
proxy.httpProxy = proxy.sslProxy = proxy.socksProxy = "PROXY_ADDRESS"
options.Proxy = proxy
options.add_argument("ignore-certificate-errors")
driver = webdriver.Chrome('/Users/nisaf/chromedriver',
chrome_options=options)
Second
proxy = "PROXY_ADDRESS"
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--proxy-server={}'.format(proxy))
driver = webdriver.Chrome(chrome_options=chrome_options)
driver.get('http://www.test.com/proxy-test')```
thanx in advance
I m trying to open the webpage in Tor Browser using Python
Code:
# Start :Code For TOR Browser
from selenium import webdriver
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
# path to TOR binary
binary = FirefoxBinary(r'C:\\Tor Browser\\Browser\\firefox.exe')
# binary = FirefoxBinary(r'C:\\Program Files (x86)\\Vidalia Bridge Bundle\\Tor\\tor.exe')
#path to TOR profile
profile = FirefoxProfile(r'C:\\Tor Browser\\Browser\\TorBrowser\\Data\\Browser\\profile.default')
# cap = DesiredCapabilities().FIREFOX
# cap["marionette"] = False
# driver = webdriver.Firefox(firefox_binary= binary, executable_path="C:\\Python\\scrapy-master\\Projects\\kgooglecom\\WithScreenshot\\geckodriver.exe")
driver = webdriver.Firefox(firefox_profile= profile, firefox_binary= binary, executable_path="C:\\Python\\scrapy-master\\Projects\\kgooglecom\\WithScreenshot\\geckodriver.exe")
# driver = webdriver.Firefox(firefox_profile=profile, firefox_binary= binary, executable_path=r"C:\\Python\\scrapy-master\\Projects\\kgooglecom\\WithScreenshot\\geckodriver.exe")
driver.get("https://www.google.com")
driver.save_screenshot("screenshot1.png")
driver.quit()
# End :Code For TOR Browser
I Use this link to download the geckodriver
Error:
Please suggest me how to resolve this error
To open a webpage through Tor Browser using Python you can use the Firefox Profile and the tor daemon and you can use the following solution:
Code Block:
from selenium import webdriver
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
import os
torexe = os.popen(r'C:\Users\AtechM_03\Desktop\Tor Browser\Browser\TorBrowser\Tor\tor.exe')
profile = FirefoxProfile(r'C:\Users\AtechM_03\Desktop\Tor Browser\Browser\TorBrowser\Data\Browser\profile.default')
profile.set_preference('network.proxy.type', 1)
profile.set_preference('network.proxy.socks', '127.0.0.1')
profile.set_preference('network.proxy.socks_port', 9050)
profile.set_preference("network.proxy.socks_remote_dns", False)
profile.update_preferences()
driver = webdriver.Firefox(firefox_profile= profile, executable_path=r'C:\Utility\BrowserDrivers\geckodriver.exe')
driver.get("http://check.torproject.org")
Browser Snapshot:
How can I use proxy server using selenium and google chrome? I attached the code and I'm not sure if this will change the actual proxy server.
# selenium imports
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import random
PROXY ="88.157.149.250:8080";
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=%s' % PROXY)
# //a[starts-with(#href, 'https://www.amazon.com/')]/#href
LINKS_XPATH = '//*[contains(#id,"result")]/div/div[3]/div[1]/a'
browser = webdriver.Chrome(executable_path="C:\\Users\Andrei\Downloads\chromedriver_win32\chromedriver.exe",
chrome_options=chrome_options)
browser.get(
'https://www.amazon.com/s/ref=lp_11444071011_nr_p_8_1/132-3636705-4291947?rh=n%3A3375251%2Cn%3A%213375301%2Cn%3A10971181011%2Cn%3A11444071011%2Cp_8%3A2229059011')
links = browser.find_elements_by_xpath(LINKS_XPATH)
for link in links:
href = link.get_attribute('href')
print(href)
from selenium import webdriver
PROXY = "88.157.149.250:8080" # IP:PORT or HOST:PORT
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=%s' % PROXY)
chrome = webdriver.Chrome(chrome_options=chrome_options)
chrome.get("http://google.com")
You can open the page https://www.whatismyip.com/my-ip-information/
chrome.get("https://www.whatismyip.com/my-ip-information/")
proxy = 192.168.22.1:8080
if proxy != None:
print('\nProxy ativado: ',proxy)
#chrome_options.add_argument('--proxy-server=%s' % proxy)
webdriver.DesiredCapabilities.CHROME['proxy'] = {
"httpProxy": proxy,
"ftpProxy": proxy,
"sslProxy": proxy,
"proxyType": "MANUAL",
}
webdriver.DesiredCapabilities.CHROME['acceptSslCerts']=True
print(webdriver.DesiredCapabilities.CHROME)
I researched on it but I get that solution:
from selenium import webdriver
profile = webdriver.FirefoxProfile()
profile.set_preference('network.proxy.type', 1)
profile.set_preference('network.proxy.socks', '127.0.0.1')
profile.set_preference('network.proxy.socks_port', 9050)
driver = webdriver.Firefox(profile)
driver.get('http://estoeslapollaconcebol.la')
It gives that error:
Can't load the profile. Profile Dir:
C:\Users\HPPAV1~1\AppData\Local\Temp\tmppcuwx3xd Firefox output:
None
When I try that solution.
from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
profile=webdriver.FirefoxProfile('C:\\Users\\HP PAV 15\\Desktop\\Tor Browser\\Browser\\TorBrowser\\Data\\Browser\\profile.default\\')
binary =FirefoxBinary('C:\\Users\\HP PAV 15\\Desktop\\Tor Browser\\Browser\\firefox')
#browser = binary.launch_browser(profile)
profile.set_preference('network.proxy.type', 1)
profile.set_preference('network.proxy.socks', '127.0.0.1')
profile.set_preference('network.proxy.socks_port', 9150)
browser=webdriver.Firefox( binary, profile)
browser.get("http://yahoo.com")
browser.save_screenshot("/Users/admin/Pictures/screenshot.png")
browser.close()
It gives me the following error:
Traceback (most recent call last): File
"C:/Python34/torfirstscript.py", line 10, in
browser=webdriver.Firefox( binary, profile) File "C:\Python34\lib\site-packages\selenium-2.43.0-py3.4.egg\selenium\webdriver\firefox\webdriver.py",
line 46, in init
self.NATIVE_EVENTS_ALLOWED and self.profile.native_events_enabled) AttributeError: 'FirefoxBinary' object has no attribute
'native_events_enabled'
By applying
browser=webdriver.Firefox( firefox_binary = binary, firefox_profile = profile)
I got this error:
Traceback (most recent call last):
File "C:\Python34\torfirstscript.py", line 9, in
browser=webdriver.Firefox( firefox_binary = binary, firefox_profile = >profile)
File "C:\Python34\lib\site-packages\selenium-2.43.0->py3.4.egg\selenium\webdriver\firefox\webdriver.py", line 59, in init
self.binary, timeout),
File "C:\Python34\lib\site-packages\selenium-2.43.0->py3.4.egg\selenium\webdriver\firefox\extension_connection.py", line 47, in >init
self.binary.launch_browser(self.profile)
File "C:\Python34\lib\site-packages\selenium-2.43.0->py3.4.egg\selenium\webdriver\firefox\firefox_binary.py", line 64, in launch_browser
self._wait_until_connectable()
File "C:\Python34\lib\site-packages\selenium-2.43.0-py3.4.egg\selenium\webdriver\firefox\firefox_binary.py", line 108, in _wait_until_connectable
self.profile.path, self._get_firefox_output()))
selenium.common.exceptions.WebDriverException: Message: "Can't load the profile. Profile Dir: >C:\Users\HPPAV1~1\AppData\Local\Temp\tmpig7zvx_0\webdriver-py-profilecopy Firefox output: None"
with that image as output.
A working example with Selenium and Tor on windows :
from selenium import webdriver
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
binary = FirefoxBinary(r"C:\Program Files (x86)\TorBrowser\Browser\firefox.exe")
profile = FirefoxProfile(r"C:\Program Files (x86)\TorBrowser\Browser\TorBrowser\Data\Browser\profile.default")
driver = webdriver.Firefox(profile, binary)
driver.get("http://stackoverflow.com")
driver.save_screenshot("screenshot.png")
driver.quit()
I tried something like this, and worked:
profile = webdriver.FirefoxProfile()
profile.set_preference('network.proxy.type', 1)
profile.set_preference('network.proxy.socks', '127.0.0.1')
profile.set_preference('network.proxy.socks_port', 9150)
driver = webdriver.Firefox(profile)
Open the Tor browser while you are doing this
Another simple solution is:
Create a new profile in Firefox or Chrome,
configure your browser to use Tor proxy (Set a SOCKS 5 proxy to address 127.0.0.1 port 9150), and then load that profile when you use webdriver.
Code for latest TOR installation on Windows:
from selenium import webdriver
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
binary = FirefoxBinary(r"C:\Users\<Windows User>\Desktop\Tor Browser\Browser\firefox.exe")
profile = FirefoxProfile(r"C:\Users\<Windows User>\Desktop\Tor Browser\Browser\TorBrowser\Data\Browser\profile.default")
driver = webdriver.Firefox(profile, binary)
driver.get("http://stackoverflow.com")
This is what worked for me, this doesn't use the tor browser but geckodriver
pip install selenium webdriver-manager
import asyncio
import os
import subprocess
from selenium.webdriver import Firefox
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.firefox.service import Service
from webdriver_manager.firefox import GeckoDriverManager
profile_path = os.path.expandvars(
r"%USERPROFILE%\Desktop\Tor Browser\Browser\TorBrowser\Data\Browser\profile.default"
)
options = Options()
options.set_preference("profile", profile_path)
service = Service(
# os.path.expandvars(r"%USERPROFILE%\Desktop\Tor Browser\Browser\firefox.exe"),
executable_path=GeckoDriverManager().install()
)
options.set_preference("network.proxy.type", 1)
options.set_preference("network.proxy.socks", "127.0.0.1")
options.set_preference("network.proxy.socks_port", 9050)
options.set_preference("network.proxy.socks_remote_dns", False)
async def main():
async def cleanup():
driver.quit()
print(torexe.pid)
torexe.kill()
try:
# https://stackoverflow.com/a/62686067/8608146
torexe = subprocess.Popen(
os.path.expandvars(
r"%USERPROFILE%\Desktop\Tor Browser\Browser\TorBrowser\Tor\tor.exe"
)
)
driver = Firefox(service=service, options=options)
driver.get("https://check.torproject.org")
driver.save_screenshot("screenshot.png")
except Exception as e:
print(e, type(e))
finally:
await cleanup()
if __name__ == "__main__":
asyncio.run(main())
I solved my similar problem on Windows:
from selenium import webdriver
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
binary = FirefoxBinary(r"C:\Users\<Windows User>\Desktop\Tor Browser\Browser\firefox.exe")
driver = webdriver.Firefox(firefox_binary=binary)
driver.profile.set_preference('network.proxy.type', 1)
driver.profile.set_preference('network.proxy.socks', '127.0.0.1')
driver.profile.set_preference('network.proxy.socks_port', 9051)
driver.get("http://stackoverflow.com")
As some of these methods do not work in the current Windows versions, returning a "tor failed to start" error would inform users that, in order to start the proxy, they will need tor already running before executing your script.
This is working as of 05-12-2020. You need to be running tor browser before running this script. This will run Tor in Chrome. Will do that only in incognito mode. If you remove that option it will connect through your isp.
import time
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
tor_proxy = "127.0.0.1:9150"
chrome_options = Options()
'''chrome_options.add_argument("--test-type")'''
chrome_options.add_argument('--ignore-certificate-errors')
'''chrome_options.add_argument('--disable-extensions')'''
chrome_options.add_argument('disable-infobars')
'''chrome_options.add_argument("--incognito")'''
chrome_options.add_argument('--user-data=C:\\Users\\user\\AppData\\Local\\Google\\Chrome\\User Data\\Default')
chrome_options.add_argument('--proxy-server=socks5://%s' % tor_proxy)
driver = webdriver.Chrome(executable_path='C:\\chromedriver.exe', options=chrome_options)
driver.get('https://www.google.com')
time.sleep(4)
driver.switch_to.frame(0)
driver.find_element_by_id("introAgreeButton").click()
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium import webdriver
caps = DesiredCapabilities.FIREFOX
caps['proxy'] = {
'proxyType': 'MANUAL',
'socksProxy': '127.0.0.1:9050',
'socksVersion': 5
}
driver = webdriver.Firefox(executable_path=r"C:\webdrivers\geckodriver.exe", capabilities=caps)
In my case this code is the only one that works.
Update selenium using:
pip install -U selenium
Then run your code, after starting TOR of course.
This error was acknowledged and repaired.