Develop Reference

How to accept cookies popup within #shadow-root (open) using Selenium Python

I am trying to press the accept button in a cookies popup in the website https://www.immobilienscout24.de/
Snapshot:
I understand that this requires
driver.execute_script("""return document.querySelector('#usercentrics-root')""")
But I can't trickle down the path to the accept button in order to click it. Can anyone provide some help?

This is one way (tested & working) you can click that button: please observe the imports, as well as the code after defining the browser/driver:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
chrome_options = Options()
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument('disable-notifications')
import time as t
webdriver_service = Service("chromedriver/chromedriver") ## path to where you saved chromedriver binary
browser = webdriver.Chrome(service=webdriver_service, options=chrome_options)
actions = ActionChains(browser)
url = 'https://www.immobilienscout24.at/regional/wien/wien/wohnung-kaufen'
browser.get(url)
page_title = WebDriverWait(browser, 3).until(EC.presence_of_element_located((By.CSS_SELECTOR, "a[title='Zur Homepage']")))
actions.move_to_element(page_title).perform()
parent_div = WebDriverWait(browser, 20000).until(EC.presence_of_element_located((By.ID, "usercentrics-root")))
shadowRoot = browser.execute_script("return arguments[0].shadowRoot", parent_div)
try:
button = WebDriverWait(shadowRoot, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button[data-testid='uc-accept-all-button']")))
button.click()
print('clicked')
except Exception as e:
print(e)
print('no click button')
That page is reacting to user's behavior, and it will only fully load the page once it detects mouse movements, hence the ActionChains() part of the code. After that, we drill down into the shadow root element, we locate the button (using Waits, to make sure it's clickable), and then we click it.
Selenium documentation can be found at https://www.selenium.dev/documentation/

The element Alle akzeptieren within the website is located within a #shadow-root (open).
Solution
To click on the element Alle akzeptieren you have to use shadowRoot.querySelector() and you can use the following Locator Strategy:
Code Block:
driver.execute("get", {'url': 'https://www.immobilienscout24.de/'})
time.sleep(10)
item = driver.execute_script('''return document.querySelector('div#usercentrics-root').shadowRoot.querySelector('button[data-testid="uc-accept-all-button"]')''')
item.click()

How to login within Tradingview site using Selenium and Python

I'm new to Python. To enter the Tradingview.com site with Selenium library.
I wrote the following code and used Xpath and CSS selector to give the address, and the Click method, but it does not work properly. Has anyone solved this problem?
import time
from selenium import webdriver
from webdriver_manager.firefox import GeckoDriverManager
driver = webdriver.Firefox(executable_path=GeckoDriverManager().install())
longInfo = ["xxx.gmail.com", "11"]
try:
driver.get("https://www.tradingview.com/#signin")
driver.set_page_load_timeout(20)
driver.maximize_window()
# click email Button for logging page
driver.find_element_by_xpath("/html/body/div[7]/div/div[2]/div/div/div/div/div/div/div[1]/div[4]/div/span").click()
time.sleep(5)
driver.find_element_by_css_selector(
"#email-signin__user-name-input__e07a4b49-2f94-4b3e-a3f8-934a5744fe02").send_keys(longInfo[0])
driver.find_element_by_css_selector(
"#email-signin__password-input__e07a4b49-2f94-4b3e-a3f8-934a5744fe02").send_keys(longInfo[1])
# click sign in Button
driver.find_element_by_xpath(
"/html/body/div[7]/div/div[2]/div/div/div/div/div/div/form/div[5]/div[2]/button/span[2]").click()
input("type for exit")
driver.quit()
except Exception as e:
print(e)
driver.quit()

It seems the locator you have used its dynamic, you need to identify the element with better approach.
Its required synchronization time while navigating.
Use explicit wait and wait for element to be clickable.
longInfo = ["xxx.gmail.com", "11"]
driver.get("https://www.tradingview.com/#signin")
wait=WebDriverWait(driver, 10)
wait.until(EC.element_to_be_clickable((By.XPATH, "//span[text()='Email']"))).click()
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "[name='username']"))).send_keys(longInfo[0])
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "[name='password']"))).send_keys(longInfo[1])
wait.until(EC.element_to_be_clickable((By.XPATH, "//button[.//span[contains(., 'Sign in')]]"))).click()
Use following libraries.
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

These values of the id attribute:
email-signin__user-name-input__e07a4b49-2f94-4b3e-a3f8-934a5744fe02
email-signin__password-input__e07a4b49-2f94-4b3e-a3f8-934a5744fe02
are dynamically generated and would change everytime you access the webpage afresh. Instead you need to use locator strategies based on static attributes.
Solution
To login within Trading View website you need to induce WebDriverWait for the element_to_be_clickable() and you can use the following locator strategies:
driver.get("https://www.tradingview.com/#signin")
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//span[text()='Email']"))).click()
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//input[#name='username']"))).send_keys("matin_mhz#stackoverflow.com")
driver.find_element(By.XPATH, "//input[#name='password']").send_keys("matin_mhz" + Keys.RETURN)
Note: You have to add the following imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC

Why does selenium produce target element is not interactable and could not be clicked error when there is an explicit wait

I am working with Selenium, I have looked on SO and have seen many posts but I am unable to implement them in a way which works. I am using the below code but the error message I am getting is:
Message: The target element is not interactable and could not be clicked
I thought it was because the commands there was no 'slack' between commands so I inserted an implicit wait, however I am still getting the same error. Any thoughts would be appreciated.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
driver = webdriver.Safari()
driver.get("https://securities.stanford.edu/filings-case.html?id=107866")
button = driver.find_element_by_link_text('Log In')
button.click()
driver.implicitly_wait(10)
username = driver.find_element_by_id("login_email")
password = driver.find_element_by_id("login_pass")
username.send_keys("EMAIL")
password.send_keys("PASSWORD")
driver.implicitly_wait(10)
driver.find_element_by_link_text('Log In').click()

There are 7 elements on that page with text Log In, this is why the following command button = driver.find_element_by_link_text('Log In') will get the first element matching this text while this is not the eolement you are looking for. So, you need to use better locator to match the correct element.
Also, you are defining driver.implicitly_wait(10) AFTER that command.
Also, it's preferably to use explicit waits of expected conditions rather than implicitly waits.
Also, if you are using implicitly wait there is no need to define it several times. This parameter is set per driver session and in most cases no need to re-set it to another value during the session.
This should work better:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Safari()
wait = WebDriverWait(driver, 20)
driver.get("https://securities.stanford.edu/filings-case.html?id=107866")
wait.until(EC.visibility_of_element_located((By.XPATH, "//a[#href="#myModalLogin"]//strong[text()='Log In']"))).click()
username = wait.until(EC.visibility_of_element_located((By.ID, "login_email")))
password = wait.until(EC.visibility_of_element_located((By.ID, "login_pass")))
username.send_keys("EMAIL")
password.send_keys("PASSWORD")
driver.find_element_by_link_text('Log In').click()

The WebElement is a <button> element but not a <a> element. Hence you can't use link_text('Log In').
You can use the following Locator Strategies
CSS_SELECTOR:
driver.get("https://securities.stanford.edu/filings-case.html?id=107866")
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div.hidden-tablet a[href='#myModalLogin']>strong"))).click()
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "input#login_email"))).send_keys("Jay#Dee.com")
driver.find_element(By.CSS_SELECTOR, "input#login_pass").send_keys("Jay#Dee.com")
driver.find_element(By.CSS_SELECTOR, "button[onclick='return submitLogin();']").click()
XPATH:
driver.get("https://securities.stanford.edu/filings-case.html?id=107866")
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//div[contains(#class, 'hidden-tablet')]//a[#href='#myModalLogin']/strong"))).click()
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//input[#id='login_email']"))).send_keys("Jay#Dee.com")
driver.find_element(By.XPATH, "//input[#id='login_pass']").send_keys("Jay#Dee.com")
driver.find_element(By.XPATH, "//button[#onclick='return submitLogin();']").click()
Note : You have to add the following imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
Browser Snapshot:

Parsing a dynamically loaded webpage with Selenium

I'm trying to parse https://www.flashscore.com/football/albania/ using Selenium in Python, but my webdriver often doesn't wait for the scores to finish loading.
Here's the code:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
driver = webdriver.Firefox()
driver.get("https://www.flashscore.com/football/albania/")
try:
WebDriverWait(driver, 100).until(
lambda s: s.execute_script("return jQuery.active == 0"))
print(driver.page_source)
finally:
driver.quit()
Occasionally, this will print out source code for a flashscore page with a blank table (i.e. the driver does not wait for the scores to finish loading). I suspect that this is because some of the live scores on the page are dynamically loaded. Is there any way to improve my wait condition?

There's an accept cookies button, so we have to click on that first.
I am using Explicit waits, first presence of table and then visibility of it's main body.
Code :
driver.maximize_window()
driver.implicitly_wait(30)
wait = WebDriverWait(driver, 30)
driver.get("https://www.flashscore.com/football/albania/")
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button#onetrust-accept-btn-handler"))).click()
try:
wait.until(EC.presence_of_element_located((By.ID, "live-table")))
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "section.event")))
print(driver.page_source)
finally:
driver.quit()
Imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
Output is certainly to long, so I would be able to post it here because stackoverflow won't allow me to do so.

How to click load more button in selenium untill page endspython?

Using selenium to load and page and need to click load more button, but couldn't able to do that.
Tried this :
from selenium import webdriver
import pandas as pd
driver = webdriver.Chrome('/Users/1/chromedriver.exe')
driver.get('https://simpletire.com/catalog?select=1&brand=61&query=catalog')
driver.find_element_by_css_selector(".btn.btn-primary.btn-lg").click();
tried the above but button is clicking and there are multiple load more how to load them multiple times untill page gets over
Error :
Tried to keep it in loop but getting :
element not interactable

The solution that worked for me is a little simple and requires a little effort but works fine.
count=20
while count>1:
button=driver.find_element_by_css_selector("button.ipl-load-more__button")
button.click()
count-=1
time.sleep(2)
//do you work once all the pages are loaded
The only thing you need to worry about is setting the right count value, if its too small you might get thrown an error, just catch it and increase/decrease the count value according to your requirement. I hope this helps.

Here is the code that should work. I am not sure how many tires available, the script ran successfully to load ~1000 results.
I have given the option to stop loading after meeting tires count, rather iterating 100+ times.
url = 'https://simpletire.com/catalog?select=1&brand=61&query=catalog'
driver.get(url)
loadingButton = WebDriverWait(driver,30).until(EC.presence_of_element_located((By.XPATH,"//div[#id='load_button']")))
maxTires = 200;
while loadingButton:
loadingButton.click()
time.sleep(2)
WebDriverWait(driver,30).until(EC.presence_of_element_located((By.XPATH,"//div[#id='is_loading'][contains(#style,'none')]")))
loadElems = driver.find_elements_by_xpath("//div[#id='load_button'][contains(#style,'block')]")
if len(loadElems)>0:
loadingButton = driver.find_element_by_xpath("//div[#id='load_button'][contains(#style,'block')]")
tiresLoaded = len(driver.find_elements_by_css_selector(".catResultWrapper.result"))
else:
print("Loaded all the tires")
break
if tiresLoaded >= maxTires:
print (tiresLoaded + " are loaded successfully.")
break

To click() on the element with text as LOAD MORE RESULTS you need to induce WebDriverWait for the desired element_to_be_clickable() and you can use the following Locator Strategies:
Code Block A:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("start-maximized")
# chrome_options.add_argument('disable-infobars')
driver = webdriver.Chrome(options=chrome_options, executable_path=r'C:\Utility\BrowserDrivers\chromedriver.exe')
driver.get("https://simpletire.com/catalog?select=1&brand=61&query=catalog")
while True:
try:
# WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//button[#class='btn btn-primary btn-lg']//span[#class='glyphicon glyphicon-play']"))).click()
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//button[contains(., 'Load More Results')]"))).click()
print("LOAD MORE RESULTS button clicked")
except TimeoutException:
print("No more LOAD MORE RESULTS button to be clicked")
break
driver.quit()
Code Block B:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("start-maximized")
# chrome_options.add_argument('disable-infobars')
driver = webdriver.Chrome(options=chrome_options, executable_path=r'C:\Utility\BrowserDrivers\chromedriver.exe')
driver.get("https://simpletire.com/catalog?select=1&brand=61&query=catalog")
while True:
try:
# ActionChains(driver).move_to_element(WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//button[#class='btn btn-primary btn-lg']//span[#class='glyphicon glyphicon-play']")))).pause(3).click().perform()
ActionChains(driver).move_to_element(WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//button[contains(., 'Load More Results')]")))).pause(5).click().perform()
print("LOAD MORE RESULTS button clicked")
except TimeoutException:
print("No more LOAD MORE RESULTS button to be clicked")
break
driver.quit()
Console Output:
LOAD MORE RESULTS button clicked
LOAD MORE RESULTS button clicked
LOAD MORE RESULTS button clicked
LOAD MORE RESULTS button clicked
LOAD MORE RESULTS button clicked
.
.
.
Browser Snapshot: