Develop Reference

python selenium chome - add_extension attribute not working for headless mode?

When I try to add an extension and retrieve some dynamic content from a website, I get an empty string. It works fine when I comment out the add_extension line. I couldn't figure out why the innerHTML for that ID would be blank.
from xvfbwrapper import Xvfb
from selenium import webdriver
from selenium.webdriver.common.by import By
import selenium.webdriver.support.ui as ui
from selenium.webdriver.chrome.options import Options
xf = Xvfb() # xf = Xvfb(1920, 1080) - will create virtual display with 1920x1080 size
xf.start()
# browser won't appear
#options = Selenium::WebDriver::Chrome::Options.new
#options = webdriver.ChromeOptions()
options = Options()
prefs = {"download.default_directory" : "/home/ritik/pes"};
options.add_experimental_option("prefs",prefs);
options.add_extension('./ghostery.crx')
driver = webdriver.Chrome(options=options)
driver.get("https://amiunique.org/fp")
wait = ui.WebDriverWait(driver, 60, 10)
wait.until(lambda driver: driver.find_element(By.ID, "summaryRes"))
fp = driver.find_element(By.ID, "summaryRes")
print(fp.text)
print(fp.text) return an empty string but in the browser, it has a string that gets populated after 5-6 seconds.

Selenium printing error The selected printer could not be found

This is in reference to the solution provided by #lifeiscomplex How to handle Firefox print dialog box in Selenium
The code is:
from time import sleep
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.firefox.options import FirefoxProfile
driver_path = 'geckodriver.exe'
firefox_options = Options()
firefox_options.add_argument("--disable-infobars")
firefox_options.add_argument("--disable-extensions")
firefox_options.add_argument("--disable-popup-blocking")
profile_options = FirefoxProfile()
user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11.5; rv:90.0) Gecko/20100101 Firefox/90.0'
profile_options.set_preference('profile_options = FirefoxProfile()', user_agent)
profile_options.set_preference("print_printer", "Mozilla Save to PDF")
profile_options.set_preference("print.always_print_silent", True)
profile_options.set_preference("print.show_print_progress", True)
profile_options.set_preference('print.save_as_pdf.links.enabled', True)
profile_options.set_preference("print.printer_Mozilla_Save_to_PDF.print_to_file", True)
# set your own file path
profile_options.set_preference('print.printer_Mozilla_Save_to_PDF.print_to_file.print_to_filename',
"testprint.pdf")
driver = webdriver.Firefox(executable_path=driver_path, options=firefox_options,
firefox_profile=profile_options)
URL = 'https://finance.yahoo.com/'
driver.get(URL)
search_field_id = 'yfin-usr-qry'
element_search_field = driver.find_element_by_id(search_field_id)
element_search_field.clear()
element_search_field.send_keys('TSLA')
element_search_field.send_keys(Keys.ENTER)
driver.execute_script("window.print()")
sleep(20)
driver.quit()
When i run this code, the code executes itself does not produce any error but i get this error dialogbox in firefox. Even if you press Ctrl+P you will get the same dialog box:
However if manually clicked on print from application menu in FireFox during the page loading process i do get the print dialog:
Could you please advise why am i getting that dialogbox.
I did change the toggle to True before running the script, which shouldn't matter since since its already in profile_options.set_preference("print.printer_Mozilla_Save_to_PDF.print_to_file", True):
Edit:
I was able to get the print dialog box to start reappearing by commenting out this line:
profile_options.set_preference("print.always_print_silent", True)
But i am not able to find the file testpage.pdf which is suppose to be generated.
My system information
----------------------------------------
Platform: Windows
OS: 10
Python: 3.8.8
Selenium: 3.141.0
Firefox: 90.0.2
Geckodriver: 0.29.0
----------------------------------------

In the comments of this question "How to handle Firefox print dialog box in Selenium" you mentioned that you had changed this line from my answer:
profile_options.set_preference('print.printer_Mozilla_Save_to_PDF.print_to_file.print_to_filename', "testprint.pdf")
to this:
profile_options.set_preference('print.printer_Microsoft_Print_to_PDF.print_to_filename', "testprint.pdf")
If you want to change that line you would have to change at least one more in the code you posted.
from time import sleep
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.firefox.options import FirefoxProfile
driver_path = 'geckodriver.exe'
firefox_options = Options()
firefox_options.add_argument("--disable-infobars")
firefox_options.add_argument("--disable-extensions")
firefox_options.add_argument("--disable-popup-blocking")
profile_options = FirefoxProfile()
user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11.5; rv:90.0) Gecko/20100101 Firefox/90.0'
profile_options.set_preference('profile_options = FirefoxProfile()', user_agent)
# this line was changed
profile_options.set_preference("print_printer", "Microsoft Print to PDF")
profile_options.set_preference("print.always_print_silent", True)
profile_options.set_preference("print.show_print_progress", True)
profile_options.set_preference('print.save_as_pdf.links.enabled', True)
# this line was changed
profile_options.set_preference("print.printer_Microsoft_Print_to_PDF.print_to_file", True)
# this line was changed
profile_options.set_preference('print.printer_Microsoft_Print_to_PDF.print_to_filename', "testprint.pdf")
driver = webdriver.Firefox(executable_path=driver_path, options=firefox_options,
firefox_profile=profile_options)
URL = 'https://finance.yahoo.com/'
driver.get(URL)
search_field_id = 'yfin-usr-qry'
element_search_field = driver.find_element_by_id(search_field_id)
element_search_field.clear()
element_search_field.send_keys('TSLA')
element_search_field.send_keys(Keys.ENTER)
driver.execute_script("window.print()")
sleep(20)
driver.quit()
There might be another line that needs to be changed, but I cannot test the code above to verify this, because I'm using macOS and not Windows 10.
You also mentioned that you commented this line out:
profile_options.set_preference("print.always_print_silent", True)
commenting out this line will launch the print dialog as showed in the graphic below.

Python Selenium Clear Chrome Cookies

I'm trying to clear my chrome cookies, but i can't click a button: BUTTON
import os
import time
from selenium.webdriver.chrome.options import Options
from selenium import webdriver
opt = Options()
opt.add_argument("start-maximized")
executable_path = r'chromedriver'
os.environ["webdriver.chrome.driver"] = executable_path
global driver
driver = webdriver.Chrome(r'C:\Users\SAMSUNG\Desktop\chromedriver', options=opt)
driver.get("https://www.youtube.com")
driver.get("chrome://settings/clearBrowserData")
time.sleep(2)
driver.find_element_by_css_selector('#clearBrowsingDataConfirm').click()

The problem comes form the chained shadowroots that doesn't allow for the element to be found:
driver.get("https://www.youtube.com")
driver.get("chrome://settings/clearBrowserData")
time.sleep(5)
javaScript = "document.querySelector('settings-ui').shadowRoot.querySelector('settings-main').shadowRoot.querySelector('settings-basic-page').shadowRoot.querySelector('settings-section > settings-privacy-page').shadowRoot.querySelector('settings-clear-browsing-data-dialog').shadowRoot.querySelector('#clearBrowsingDataDialog').querySelector('#clearBrowsingDataConfirm').click()"
driver.execute_script(javaScript)

Using python selenium to click and download files

i'm currently trying to automate the task of clicking different download links from this website:
https://www.theice.com/clear-us/risk-management#margin-rates
in this page, i first have to click the "Download ICE Risk Model array files" header which gives me 2 dropdowns from which i want to first click the "Final" link which downloads a csv file for each month of each available year.
Currently, both the dropdowns change due to hidden dropdown menus above, i have first tried to make them visible which was successful as well as changing year in it using selenium click,
The problem arising is that i'm not able to click the "Final" link in the csv section but it just clicks the "
Intercontinental Exchange"
button in the footer and navigates to a new page.
Is there anyway to get this task done ?
As well as is it possible to change the download location to the current directory where the .py script is ?
This is the python code so far, i currently removed the headless part to see what's going on :
from bs4 import BeautifulSoup
import requests
from selenium.webdriver.chrome.options import Options
from shutil import which
from selenium.webdriver.common.keys import Keys
from selenium import webdriver
from selenium.webdriver.support.ui import Select
import time
import os
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--log-level=3")
chrome_path = which("chromedriver")
driver = webdriver.Chrome(executable_path=chrome_path, options=chrome_options)
driver.set_window_size(1366, 768)
driver.get("https://www.theice.com/clear-us/risk-management#margin-rates")
main_button = driver.find_element_by_xpath('//h4[#class="collapsible-section-header"]')
main_button.click()
time.sleep(5)
driver.execute_script("document.getElementById('icus-ice-form-year').style.display = 'block';")
driver.execute_script("document.getElementById('icus-ice-form-month').style.display = 'block';")
time.sleep(1)
dropdown_1 = Select(driver.find_element_by_xpath('//select[#id="icus-ice-form-year"]'))
dropdown_2 = Select(driver.find_element_by_xpath('//select[#id="icus-ice-form-year"]'))
main_table_div = driver.find_element_by_xpath('//div[#id="icus-ice-riskarraytable"]')
main_table = main_table_div.find_element_by_xpath('//table[#class="table table-data"]')
for opt in dropdown_1.options:
opt.click()
for opt2 in dropdown_2.options:
opt2.click()
time.sleep(3)
download_links_1 = main_table.find_elements_by_xpath('//td[#class="table-partitioned"]')
for dow in download_links_1:
try:
temp_dow = dow.find_element_by_xpath('//a')
temp_dow.click()
time.sleep(4)
except:
pass

This should switch the downloads to the current working directory and print all the early csvs. change '//[#id="icus-ice-riskarraytable"]/table/tbody/tr[{0}]/td[2]/a' to '//[#id="icus-ice-riskarraytable"]/table/tbody/tr[{0}]/td[3]/a' for the other final csv.
options = Options()
currentDirectory = os.getcwd()
prefs = {
"download.default_directory": currentDirectory,
"download.prompt_for_download": False
}
#print(currentDirectory)
options.add_experimental_option("prefs", prefs)
driver.get("https://www.theice.com/clear-us/risk-management#margin-rates")
driver.implicitly_wait(5)
main_button = driver.find_element_by_xpath('//h4[#class="collapsible-section-header"]')
main_button.click()
driver.implicitly_wait(5)
driver.execute_script("document.getElementById('icus-ice-form-year').style.display = 'block';")
driver.execute_script("document.getElementById('icus-ice-form-month').style.display = 'block';")
driver.implicitly_wait(5)
drop1length=len(driver.find_elements_by_xpath('//select[#id="icus-ice-form-year"]/option'))
#print(drop1length)
for i in range(1,drop1length-1):
drop1=Select(driver.find_element_by_xpath('//select[#id="icus-ice-form-year"]'))
drop1.select_by_index(i)
drop2length=len(driver.find_elements_by_xpath('//select[#id="icus-ice-form-month"]/option'))
#print(drop2length)
for j in range(1,drop2length-1):
drop2=Select(driver.find_element_by_xpath('//select[#id="icus-ice-form-month"]'))
driver.implicitly_wait(5)
drop2.select_by_index(j)
download_links_length = len(driver.find_elements_by_xpath('//*[#id="icus-ice-riskarraytable"]/table/tbody/tr/td[2]/a'))
#print(download_links_length)
for dow in range(1,download_links_length-1):
try:
element = driver.find_element_by_xpath('//*[#id="icus-ice-riskarraytable"]/table/tbody/tr[{0}]/td[2]/a'.format(str(dow)))
driver.implicitly_wait(5)
driver.execute_script("arguments[0].click();", element)
driver.switch_to.window(driver.window_handles[0])
except Exception as e:
print(e)
Import these
import os
from selenium.webdriver.chrome.options import Options

PhantomJS returning empty web page (python, Selenium)

Trying to screen scrape a web site without having to launch an actual browser instance in a python script (using Selenium). I can do this with Chrome or Firefox - I've tried it and it works - but I want to use PhantomJS so it's headless.
The code looks like this:
import sys
import traceback
import time
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
dcap = dict(DesiredCapabilities.PHANTOMJS)
dcap["phantomjs.page.settings.userAgent"] = (
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/53 "
"(KHTML, like Gecko) Chrome/15.0.87"
)
try:
# Choose our browser
browser = webdriver.PhantomJS(desired_capabilities=dcap)
#browser = webdriver.PhantomJS()
#browser = webdriver.Firefox()
#browser = webdriver.Chrome(executable_path="/usr/local/bin/chromedriver")
# Go to the login page
browser.get("https://www.whatever.com")
# For debug, see what we got back
html_source = browser.page_source
with open('out.html', 'w') as f:
f.write(html_source)
# PROCESS THE PAGE (code removed)
except Exception, e:
browser.save_screenshot('screenshot.png')
traceback.print_exc(file=sys.stdout)
finally:
browser.close()
The output is merely:
<html><head></head><body></body></html>
But when I use the Chrome or Firefox options, it works fine. I thought maybe the web site was returning junk based on the user agent, so I tried faking that out. No difference.
What am I missing?
UPDATED: I will try to keep the below snippet updated with until it works. What's below is what I'm currently trying.
import sys
import traceback
import time
import re
from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.support import expected_conditions as EC
dcap = dict(DesiredCapabilities.PHANTOMJS)
dcap["phantomjs.page.settings.userAgent"] = (
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/53 (KHTML, like Gecko) Chrome/15.0.87")
try:
# Set up our browser
browser = webdriver.PhantomJS(desired_capabilities=dcap, service_args=['--ignore-ssl-errors=true'])
#browser = webdriver.Chrome(executable_path="/usr/local/bin/chromedriver")
# Go to the login page
print "getting web page..."
browser.get("https://www.website.com")
# Need to wait for the page to load
timeout = 10
print "waiting %s seconds..." % timeout
wait = WebDriverWait(browser, timeout)
element = wait.until(EC.element_to_be_clickable((By.ID,'the_id')))
print "done waiting. Response:"
# Rest of code snipped. Fails as "wait" above.

I was facing the same problem and no amount of code to make the driver wait was helping.
The problem is the SSL encryption on the https websites, ignoring them will do the trick.
Call the PhantomJS driver as:
driver = webdriver.PhantomJS(service_args=['--ignore-ssl-errors=true', '--ssl-protocol=TLSv1'])
This solved the problem for me.

You need to wait for the page to load. Usually, it is done by using an Explicit Wait to wait for a key element to be present or visible on a page. For instance:
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
# ...
browser.get("https://www.whatever.com")
wait = WebDriverWait(driver, 10)
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "div.content")))
html_source = browser.page_source
# ...
Here, we'll wait up to 10 seconds for a div element with class="content" to become visible before getting the page source.
Additionally, you may need to ignore SSL errors:
browser = webdriver.PhantomJS(desired_capabilities=dcap, service_args=['--ignore-ssl-errors=true'])
Though, I'm pretty sure this is related to the redirecting issues in PhantomJS. There is an open ticket in phantomjs bugtracker:
PhantomJS does not follow some redirects

driver = webdriver.PhantomJS(service_args=['--ignore-ssl-errors=true', '--ssl-protocol=TLSv1'])
This worked for me