Develop Reference

selenium python how to set new proxy for every request without opining new browser?

Every time I opened an new browser in my for loop for set new proxy but is there any way I can set new proxy in every request without opening the new browser. I want browser will be open once then it will be set new proxy for every loop.
here is my code:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium_stealth import stealth
from selenium.webdriver.common.action_chains import ActionChains
import time, csv
from proxy import *
ip_list = []
free_proxy_list_func(ip_list) # this function scraping proxy from website
options = Options()
options.add_argument("start-maximized")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
options.add_argument("--disable-blink-features=AutomationControlled")
driver = webdriver.Chrome(options=options,desired_capabilities = proxy_func(ip_list))
stealth(driver,
languages=["en-US", "en"],
vendor="Google Inc.",
platform="Win32",
webgl_vendor="Intel Inc.",
renderer="Intel Iris OpenGL Engine",
fix_hairline=True,
)
file1 = open('url.txt', 'r')
Lines = file1.readlines()
for i in Lines:
product_link = i
driver = webdriver.Chrome(options=options,desired_capabilities = proxy_func(ip_list)) #proxy_func(ip_list) this my proxy function
driver.get(i)
if write driver = webdriver.Chrome(options=options,desired_capabilities = proxy_func(ip_list)) outside of my for loop then it will not set new proxy on every url request.
here is my proxy function:
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.common.proxy import Proxy, ProxyType
def proxy_func(ip_list):
print('total number of ip',len(ip_list))
item = random.choice(tuple(ip_list))
print("ip: ",item)
proxy_ip_port = item
proxy = Proxy()
proxy.proxy_type = ProxyType.MANUAL
proxy.http_proxy = proxy_ip_port
proxy.ssl_proxy = proxy_ip_port
capabilities = webdriver.DesiredCapabilities.CHROME
proxy.add_to_capabilities(capabilities)

Not able to set proxy in selenium Python

I have the following bit of code:
from selenium import webdriver
from selenium.webdriver import Firefox
from selenium.webdriver.common.keys import Keys
from selenium.webdriver import ActionChains
from selenium.webdriver.common.proxy import Proxy, ProxyType
proxy = Proxy({
'proxyType': ProxyType.MANUAL,
'httpProxy': '192.156.1.1:33',
'ftpProxy': '192.156.1.1:33',
'sslProxy': '192.156.1.1:33',
'noProxy': '' # set this value as desired
})
url = 'http://www.expressvpn.com/what-is-my-ip'
driver_path = 'C:\\Users\\user\\geckodriver.exe'
browser = Firefox(executable_path = driver_path, proxy = proxy)
browser.get(url)
For some reason everytime i check the ip, it is showing my true IP and not the proxy IP. Why is it doing that and could you please advise how this can be accomplished? Is there some problem with the code?

I started looking into this and noted that proxies are set using WebDriver capabilities and proxy configurations in the geckodriver.
I used proxy information for these sources from testing.
Free proxy lists:
free-proxy.cz
Geonode
Please let me point that using free proxy IP addresses can be highly problematic. These type of proxies are notorious for having connections issues, such as timeouts related to latency. Plus these sites can also be intermittent, which means that they can go down at anytime. And sometimes these sites are being abused, so they can get blocked.
The code below uses DesiredCapabilities with selenium.
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.firefox.options import FirefoxProfile
from selenium.webdriver.firefox.options import DesiredCapabilities
firefox_options = Options()
firefox_options.add_argument("--disable-infobars")
firefox_options.add_argument("--disable-extensions")
firefox_options.add_argument("--disable-popup-blocking")
profile_options = FirefoxProfile()
user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11.5; rv:90.0) Gecko/20100101 Firefox/90.0'
firefox_options.set_preference('profile_options = FirefoxProfile()', user_agent)
firefox_capabilities = DesiredCapabilities().FIREFOX
firefox_capabilities['proxy'] = {
"proxyType": "MANUAL",
"sslProxy": '34.95.40.165:3128',
}
driver = webdriver.Firefox(executable_path='/usr/local/bin/geckodriver', options=firefox_options, desired_capabilities=firefox_capabilities)
URL = 'http://www.expressvpn.com/what-is-my-ip'
driver.get(URL)
You can also do it this way:
from selenium import webdriver
from selenium.webdriver.common.proxy import Proxy
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.firefox.options import FirefoxProfile
from selenium.webdriver.firefox.options import DesiredCapabilities
firefox_options = Options()
firefox_options.add_argument("--disable-infobars")
firefox_options.add_argument("--disable-extensions")
firefox_options.add_argument("--disable-popup-blocking")
profile_options = FirefoxProfile()
user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11.5; rv:90.0) Gecko/20100101 Firefox/90.0'
firefox_options.set_preference('profile_options = FirefoxProfile()', user_agent)
firefox_capabilities = DesiredCapabilities().FIREFOX
firefox_proxies = Proxy()
firefox_proxies.ssl_proxy = '143.110.148.15:8080'
firefox_proxies.add_to_capabilities(firefox_capabilities)
driver = webdriver.Firefox(executable_path='/usr/local/bin/geckodriver', options=firefox_options,
desired_capabilities=firefox_capabilities)
URL = 'http://www.expressvpn.com/what-is-my-ip'
driver.get(URL)
You can also use the Python package http_request_randomize to obtain a proxy IP address, which can be passed to the geckodriver.
import random
import logging
from selenium import webdriver
from selenium.webdriver.common.proxy import Proxy
from selenium.webdriver.firefox.options import Options
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.firefox.options import FirefoxProfile
from selenium.webdriver.firefox.options import DesiredCapabilities
from http_request_randomizer.requests.proxy.ProxyObject import Protocol
from http_request_randomizer.requests.proxy.requestProxy import RequestProxy
# Obtain a list of HTTPS proxies
# Suppress the console debugging output by setting the log level
req_proxy = RequestProxy(log_level=logging.ERROR, protocol=Protocol.HTTPS)
# Obtain a random single proxy from the list of proxy addresses
random_proxy = random.sample(req_proxy.get_proxy_list(), 1)
firefox_options = Options()
firefox_options.add_argument("--disable-infobars")
firefox_options.add_argument("--disable-extensions")
firefox_options.add_argument("--disable-popup-blocking")
profile_options = FirefoxProfile()
user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11.5; rv:90.0) Gecko/20100101 Firefox/90.0'
firefox_options.set_preference('profile_options = FirefoxProfile()', user_agent)
firefox_capabilities = DesiredCapabilities().FIREFOX
# add the random proxy to firefox_capabilities
firefox_proxies = Proxy()
firefox_proxies.ssl_proxy = random_proxy[0].get_address()
firefox_proxies.add_to_capabilities(firefox_capabilities)
driver = webdriver.Firefox(executable_path='/usr/local/bin/geckodriver', options=firefox_options,
desired_capabilities=firefox_capabilities)
try:
# print proxy IP for testing
print(random_proxy[0].get_address())
# output
93.183.250.200:53281
URL = 'http://www.expressvpn.com/what-is-my-ip'
driver.get(URL)
except TimeoutException as e:
print("A Page load Timeout Occurred.")
driver.quit()
As previously stated free proxy can have multiple issue. The code below shows how to use a proxy judge to check the status of an individual proxy.
import random
import logging
from time import sleep
from random import randint
from proxy_checking import ProxyChecker
from http_request_randomizer.requests.proxy.ProxyObject import Protocol
from http_request_randomizer.requests.proxy.requestProxy import RequestProxy
def random_ssl_proxy_address():
# Obtain a list of HTTPS proxies
# Suppress the console debugging output by setting the log level
req_proxy = RequestProxy(log_level=logging.ERROR, protocol=Protocol.HTTPS)
# Obtain a random single proxy from the list of proxy addresses
random_proxy = random.sample(req_proxy.get_proxy_list(), 1)
return random_proxy[0].get_address()
def get_proxy_address():
proxy_address = random_ssl_proxy_address()
checker = ProxyChecker()
proxy_judge = checker.check_proxy(proxy_address)
proxy_status = [value for key, value in proxy_judge.items() if key == 'status']
if proxy_status[0]:
return proxy_address
else:
print('Looking for a valid proxy address.')
# this sleep timer is helping with some timeout issues
# that were happening when querying
sleep(randint(5, 10))
get_proxy_address()
random_ssl_proxy = get_proxy_address()
print(f'Valid proxy address: {random_ssl_proxy}')
# output
Valid proxy address: 98.116.152.143:3128
Please note that the proxy_checker Package that I used doesn't have any embedded error handling, so you will have to add some to catch some of the errors.

How to hide and then open browser in selenium python?

I need to hide browser do some actions and then open browser in selenium python?
some code:
driver = webdriver.Chrome('./chromedriver') # connecting driver
options.add_argument('headless') # that's how I hide browser
driver = webdriver.Chrome(chrome_options=options)
driver.get("google.com")
and now I need to open browser for user

You wont able to do it with your current code as your have initiated chromedriver in headless mode and your browser simulation program that does not have a user interface.Also your url is not corrent in above example. Try below code
options = webdriver.ChromeOptions()
options.add_argument("--headless")
driver = webdriver.Chrome(executable_path=r" path of chromedriver.exe",chrome_options=options)
driver = webdriver.Chrome(executable_path=r"C:\New folder\chromedriver.exe")
base = "https://www.google.com/"
driver.get(base)
Output:

Another example
import time
from selenium import webdriver
options = webdriver.ChromeOptions()
options.add_argument("--headless")
driver = webdriver.Chrome(options=options)
headless_page = "https://www.google.com/"
driver.get(headless_page)
url = driver.current_url
print(url) # print headless url
time.sleep(2)
driver = webdriver.Chrome() # reset headless to false
driver.get(url)

How to set luminati proxy on Selenium Webdriver for chrome in python?

I want to set luminati proxy in webdriver.Chrome for selenium python.
I have tried using this following command:
from selenium import webdriver
from selenium.webdriver.common.proxy import Proxy, ProxyType
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.proxy import *
PROXY = '127.0.0.1:24000'
proxy = Proxy()
proxy.http_proxy = PROXY
proxy.ftp_proxy = PROXY
proxy.sslProxy = PROXY
proxy.no_proxy = "localhost" #etc... ;)
proxy.proxy_type = ProxyType.MANUAL
#limunati customer info
proxy.socksUsername = 'lum-customer-XXXX-zone-XXXX'
proxy.socksPassword = "XXXX"
capabilities = webdriver.DesiredCapabilities.CHROME
proxy.add_to_capabilities(capabilities)
driver = webdriver.Chrome(desired_capabilities=capabilities)
I used my Luminati username,zone and password to setup this. But it's not working.

Try this code snippet:
from selenium import webdriver
# http://username:password#localhost:8080
PROXY = "http://lum-customer-XXXX-zone-XXXX:XXXX#localhost:8080"
# Create a copy of desired capabilities object.
desired_capabilities = webdriver.DesiredCapabilities.CHROME.copy()
# Change the proxy properties of that copy.
desired_capabilities['proxy'] = {
"httpProxy":PROXY,
"ftpProxy":PROXY,
"sslProxy":PROXY,
"noProxy":None,
"proxyType":"MANUAL",
"class":"org.openqa.selenium.Proxy",
"autodetect":False
}
# you have to use remote, otherwise you'll have to code it yourself in python to
# dynamically changing the system proxy preferences
driver = webdriver.Remote("http://localhost:4444/wd/hub", desired_capabilities)
from official resource.

Try this code snippet. It's work for me with Chrome, but you can replace Chrome by Firefox :
from selenium.webdriver.chrome.options import Options
from seleniumwire import webdriver
import time
PATH = "path to driver"
options = {
'proxy':
{
'http': 'http://lum-customer-CUST_ID-zone-ZONE_NAME-country-COUNTRY_TARGET:PASSWORD#zproxy.lum-superproxy.io:22225',
'https': 'https://lum-customer-CUST_ID-zone-ZONE_NAME-country-COUNTRY_TARGET:PASSWORD#zproxy.lum-superproxy.io:22225'
},
}
driver = webdriver.Chrome(PATH, seleniumwire_options=options)
# If need to add headers to request
# driver.header_overrides = {
# }
driver.get("https://lumtest.com/myip.json")
print(driver.execute_script('return document.body.innerHTML;')) #print page html
time.sleep(3)
driver.quit()

Likely it's not working because you haven't removed setting of the proxy.no_proxy = 'localhost', that value shall be excluded.
If you haven't used Python Selenium Luminati Proxy Desired Capabilities that link before it will be useful ...

String proxi = "--your proxy url and port";// like 12345:1212
Proxy proxy = new Proxy();
proxy.setHttpProxy(proxi);
//limunati customer info
proxy.setSocksUsername("Your Username");
proxy.setSocksPassword("Your Password");
ChromeOptions options = new ChromeOptions();
options.setCapability("proxy", proxy);
System.setProperty("webdriver.chrome.driver", "Drivers/chromedriver.exe");
driver = new ChromeDriver(options);
driver.get("https://www.google.com");

Set proxy server with selenium and Chrome

How can I use proxy server using selenium and google chrome? I attached the code and I'm not sure if this will change the actual proxy server.
# selenium imports
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import random
PROXY ="88.157.149.250:8080";
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=%s' % PROXY)
# //a[starts-with(#href, 'https://www.amazon.com/')]/#href
LINKS_XPATH = '//*[contains(#id,"result")]/div/div[3]/div[1]/a'
browser = webdriver.Chrome(executable_path="C:\\Users\Andrei\Downloads\chromedriver_win32\chromedriver.exe",
chrome_options=chrome_options)
browser.get(
'https://www.amazon.com/s/ref=lp_11444071011_nr_p_8_1/132-3636705-4291947?rh=n%3A3375251%2Cn%3A%213375301%2Cn%3A10971181011%2Cn%3A11444071011%2Cp_8%3A2229059011')
links = browser.find_elements_by_xpath(LINKS_XPATH)
for link in links:
href = link.get_attribute('href')
print(href)

from selenium import webdriver
PROXY = "88.157.149.250:8080" # IP:PORT or HOST:PORT
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=%s' % PROXY)
chrome = webdriver.Chrome(chrome_options=chrome_options)
chrome.get("http://google.com")

You can open the page https://www.whatismyip.com/my-ip-information/
chrome.get("https://www.whatismyip.com/my-ip-information/")

proxy = 192.168.22.1:8080
if proxy != None:
print('\nProxy ativado: ',proxy)
#chrome_options.add_argument('--proxy-server=%s' % proxy)
webdriver.DesiredCapabilities.CHROME['proxy'] = {
"httpProxy": proxy,
"ftpProxy": proxy,
"sslProxy": proxy,
"proxyType": "MANUAL",
}
webdriver.DesiredCapabilities.CHROME['acceptSslCerts']=True
print(webdriver.DesiredCapabilities.CHROME)