Develop Reference

Finding an element and clicking on it to navigate to another webpage using selenium

I'm trying to use selenium to open up this webpage (https://app.powerbi.com/view?r=eyJrIjoiMzE2ZDIyN2YtODY1Yy00ZGY0LWE4YTktNDcxOTcwYWQyMjM5IiwidCI6IjcyMmVhMGJlLTNlMWMtNGIxMS1hZDZmLTk0MDFkNjg1NmUyNCJ9) and click on the Tram icon to navigate to the web page I want to scrape.
This is what I've tried up to now
from selenium import webdriver
driver=webdriver.Chrome()
driver.get("https://app.powerbi.com/view?r=eyJrIjoiMzE2ZDIyN2YtODY1Yy00ZGY0LWE4YTktNDcxOTcwYWQyMjM5IiwidCI6IjcyMmVhMGJlLTNlMWMtNGIxMS1hZDZmLTk0MDFkNjg1NmUyNCJ9")
driver.maximize_window()
x=driver.find_element("class name", "imageBackground")
print(x)
#driver.find_element("class name", "imageBackground").click()
However, the element returns none. I'm not sure how to find the element and click the div to navigate to the next page as well.

I've updated your code.
I added a WebDriverWait that pauses until the tram element is clickable, then it clicks on the Tram element and goes to the next page.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
if __name__ == "__main__":
driver=webdriver.Chrome()
driver.get("https://app.powerbi.com/view?r=eyJrIjoiMzE2ZDIyN2YtODY1Yy00ZGY0LWE4YTktNDcxOTcwYWQyMjM5IiwidCI6IjcyMmVhMGJlLTNlMWMtNGIxMS1hZDZmLTk0MDFkNjg1NmUyNCJ9")
driver.maximize_window()
wait = WebDriverWait(driver, 30)
tramElementXpath = '//*[#aria-label="Tram Bookmark . Click here to follow link"]'
x = wait.until(EC.element_to_be_clickable((By.XPATH, tramElementXpath)))
x.click()

How I can "save & exit" cookies popup with Selenium

I'm triying to scrape a transfermark web section and I don't want to accept all cookies in popup, but to configure it and accept these changes. I've achieve do click on "option" button in first frame, but when second frame appears I can't click on "save and exit" button.
This is my code:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
url = 'https://www.transfermarkt.es/transfers/transfertagedetail/statistik/top/land_id_zu/0/land_id_ab/0/leihe//datum/2022-07-10'
driver = webdriver.Chrome()
driver.minimize_window()
driver.get(url_base)
wait = WebDriverWait(driver, 30)
wait.until(EC.frame_to_be_available_and_switch_to_it((By.XPATH, '//*[#id="sp_message_iframe_575430"]')))
wait.until(EC.element_to_be_clickable((By.XPATH, '//*[#id="notice"]/div[3]/div[1]/button'))).click()
wait.until(EC.frame_to_be_available_and_switch_to_it((By.XPATH, '//*[#id="sp_message_iframe_225826"]')))
wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/div/div[2]/div[5]/button[2]'))).click()

To be able to switch to another iframe you need to switch back to default content first:
...
wait.until(EC.element_to_be_clickable((By.XPATH, '//*[#id="notice"]/div[3]/div[1]/button'))).click()
driver.switch_to.default_content()
wait.until(EC.frame_to_be_available_and_switch_to_it((By.XPATH, '//*[#id="sp_message_iframe_225826"]')))
...

find the URL after button click from the website using selenium python

Every Button of the website may contain the link, for the below website how to find out URL appears in next tab.
wants to print and scrape the URL after the button click
am using firefox web driver
driver.get("https://www.dove.com/us/en/skin-care/body-lotion/cream-oil-intensive-body-lotion.html")
driver.find_element_by_xpath("//span[contains(text(),'Ingredients')]").click()
time.sleep(3)
driver.find_element_by_xpath("//button[contains(text(),'Go to SmartLabel™')]").click()

This should be easy, just use driver.current_url. So with your code you could try
driver.get("https://www.dove.com/us/en/skin-care/body-lotion/cream-oil-intensive-body-lotion.html")
driver.find_element_by_xpath("//span[contains(text(),'Ingredients')]").click()
time.sleep(3)
driver.find_element_by_xpath("//button[contains(text(),'Go to SmartLabel™')]").click()
time.sleep(5)
driver.switch_to.window(driver.window_handles[1])
print(driver.current_url)

I saw few problems:
1 Waits. Get rid of time.sleep(). Replace it with explicit/implicit waits. I observed that these elements are the last that are loaded on the page: picture[class='loaded']. So, I added wait for them.
2 To switch between tabs use: driver.switch_to.window(driver.window_handles[1]), driver.switch_to.window(driver.window_handles[0]) - to switch to initial tab.
Solution for Chrome
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Chrome(executable_path='/snap/bin/chromium.chromedriver')
# driver.implicitly_wait(10)
driver.get("https://www.dove.com/us/en/skin-care/body-lotion/cream-oil-intensive-body-lotion.html")
wait = WebDriverWait(driver, 30)
wait.until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, "picture[class='loaded']")))
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, ".collapsed>a[title='Ingredients']")))
driver.find_element_by_css_selector(".collapsed>a[title='Ingredients']").click()
wait.until(EC.element_to_be_clickable((By.XPATH, "//button[contains(text(),'Go to SmartLabel')]")))
driver.find_element_by_xpath("//button[contains(text(),'Go to SmartLabel')]").click()
driver.switch_to.window(driver.window_handles[1])
print(driver.current_url)
driver.close()
driver.switch_to.window(driver.window_handles[0])
print(driver.current_url)
Output:
https://smartlabel.unileverusa.com/011111375512-0001-en-US/index.html
https://www.dove.com/us/en/skin-care/body-lotion/cream-oil-intensive-body-lotion.html
For Firefox you'll need to wait for at least one element on the second page, otherwise the output will not give you expected link:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Firefox()
driver.implicitly_wait(10)
driver.get("https://www.dove.com/us/en/skin-care/body-lotion/cream-oil-intensive-body-lotion.html")
wait = WebDriverWait(driver, 30)
wait.until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, "picture[class='loaded']")))
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, ".collapsed>a[title='Ingredients']")))
driver.find_element_by_css_selector(".collapsed>a[title='Ingredients']").click()
wait.until(EC.element_to_be_clickable((By.XPATH, "//button[contains(text(),'Go to SmartLabel')]")))
driver.find_element_by_xpath("//button[contains(text(),'Go to SmartLabel')]").click()
driver.switch_to.window(driver.window_handles[1])
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, ".container-fluid.content-section")))
print(driver.current_url)
driver.close()
driver.switch_to.window(driver.window_handles[0])
print(driver.current_url)
P.S. If you are looking for a way to find links by attribute names, there is no way because this button does not have such. The link is generated.

Selenium crashes when I'm trying to parse the next page (and seven after it) on a website. Any way to tackle this?

I want to parse an IMDb film rating located here on around 8 pages. In order to do that I'm using Selenium, and I'm having trouble with clicks, proceeding algorithm to next page. In the end I need 1000 titles when I'll continue using BeautifulSoup. Code below isn't working, I need to use button 'NEXT' with this HTML:
<a class="flat-button lister-page-next next-page" href="/list/ls000004717/?page=2">
Next
</a>
This is the code:
from selenium import webdriver as wb
browser = wb.Chrome()
browser.get('https://www.imdb.com/list/ls000004717/')
field = browser.find_element_by_name("flat-button lister-page-next next-page").click()
Error is the following:
NoSuchElementException: Message: no such element: Unable to locate element: {"method":"css selector","selector":".flat-button lister-page-next next-page"}
(Session info: chrome=78.0.3904.108)
I suppose I lack knowledge of syntax needed, or maybe I mixed it up a little. I tried searching on SO, though every example is pretty unique and I don't possess the knowledge to extrapolate these cases fully. Any way Selenium can handle that?

You could try using an XPath to query on the Next text inside the button. You should also probably invoke WebDriverWait since you are navigating across multiple pages, then scroll into view since this is at the bottom of the page:
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from time import sleep
browser = wb.Chrome()
browser.get('https://www.imdb.com/list/ls000004717/')
# keep clicking next until we reach the end
for i in range(0,9):
# wait up to 10s before locating next button
try:
next_button = WebDriverWait(browser, 10).until(EC.element_to_be_clickable((By.XPATH, "//a[contains(#class, 'page') and contains(text(), 'Next')]")))
# scroll down to button using Javascript
browser.execute_script("arguments[0].scrollIntoView(true);", next_button)
# click the button
# next_button.click() this throws exception -- replace with JS click
browser.execute_script("arguments[0].click();", next_button)
# I never recommend using sleep like this, but WebDriverWait is not waiting on next button to fully load, so it goes stale.
sleep(5)
# case: next button no longer exists, we have reached the end
except TimeoutException:
break
I also wrapped everything in a try / except TimeoutException block to handle the case where we have reached the end of pages, and Next button no longer exists, thus breaking out of the loop. This worked on multiple pages for me.
I also had to add an explicit sleep(5) because even after invoking WebDriverWait on element_to_be_clickable, next_button was still throwing StaleElementReferenceException. It seems like WebDriverWait was finishing before page was fully loaded, causing the status of next_button to change after it had been located. Normally adding sleep(5) is bad practice, but there did not seem to be another workaround here. If anyone else has a suggestion on this, feel free to comment / edit the answer.

There are a couple of ways that could work:
1. Use a selector for the next button and loop until the end:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as ec
browser = webdriver.Chrome()
browser.get('https://www.imdb.com/list/ls000004717/')
selector = 'a[class*="next-page"]'
num_pages = 10
for page in range(pages):
# Wait for the element to load
WebDriverWait(browser, 10).until(ec.presence_of_element_located((By.CSS_SELECTOR, selector)))
# ... Do rating parsing here
browser.find_element_by_css_selector(selector).click()
Instead of clicking on the element, the other option could be to navigate to the next page using broswer.get('...'):
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as ec
# Set up browser as before and navigate to the page
browser = webdriver.Chrome()
browser.get('https://www.imdb.com/list/ls000004717/')
selector = 'a[class*="next-page"]'
base_url = 'https://www.imdb.com/list/ls000004717/'
page_extension = '?page='
# Already at page = 1, so only needs to loop 9 times
for page in range(2, pages + 1):
# Wait for the page to load
WebDriverWait(browser, 10).until(ec.presence_of_element_located((By.CSS_SELECTOR, selector)))
# ... Do rating parsing here
next_page = base_url + page_extension + str(page)
browser.get(next_page)
As a note: field = browser.find_element_by_name("...").click() will not assign field to a webelement, as the click() method has no return value.

You could try a partial css selector.
browser.find_element_by_css_selector("a[class*='next-page']").click()

To click on the element with text as NEXT till the 901 - 1,000 of 1,000 page you have to:
scrollIntoView() the element once the visibility_of_element_located() is achieved.
Induce WebDriverWait for the element_to_be_clickable()
You can use the following solution:
Code Block:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
options = webdriver.ChromeOptions()
options.add_argument("start-maximized")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
driver = webdriver.Chrome(options=options, executable_path=r'C:\WebDrivers\chromedriver.exe')
driver.get('https://www.imdb.com/list/ls000004717/')
driver.execute_script("return arguments[0].scrollIntoView(true);", WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.CSS_SELECTOR, "span.pagination-range"))))
while True:
try:
WebDriverWait(driver, 20).until(EC.invisibility_of_element((By.CSS_SELECTOR, "div.row.text-center.lister-working.hidden")))
driver.execute_script("return arguments[0].scrollIntoView(true);", WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.CSS_SELECTOR, "span.pagination-range"))))
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "a.flat-button.lister-page-next.next-page"))).click()
print("Clicked on NEXT button")
except TimeoutException as e:
print("No more NEXT button")
break
driver.quit()
Console Output:
Clicked on NEXT button
Clicked on NEXT button
Clicked on NEXT button
Clicked on NEXT button
Clicked on NEXT button
Clicked on NEXT button
Clicked on NEXT button
Clicked on NEXT button
Clicked on NEXT button
No more NEXT button

Python Selenium Xpath from firebug not found

I am trying to login to the ESPN website using selenium. Here is my code thus far
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Firefox()
driver.maximize_window()
url = "http://www.espn.com/fantasy/"
driver.get(url)
login_button = driver.find_element_by_xpath("/html/body/div[6]/section/section/div/section[1]/div/div[1]/div[2]/a[2]")
login_button.click()
try:
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.XPATH, "/html/body/div[2]/div/div/section/section/form/section/div[1]/div/label/span[2]/input")))
except:
driver.quit()
Basically, there are 2 steps, first I have to click the login button and then I have to fill in the form. Currently, I am clicking the login button and the form is popping up but then I can't find the form. I have been using firebug to get the xpath as suggested in other SO questions. I don't really know much about selenium so I am not sure where to look

Try to use
driver.switch_to_frame('disneyid-iframe')
# handle authorization pop-up
driver.switch_to_default_content() # if required

This works for me, switching to the iframe first. Note that you will need to switch back out of the iframe after entering the credentials.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Firefox()
driver.maximize_window()
url = "http://www.espn.com/fantasy/"
driver.get(url)
login_button = driver.find_element_by_xpath("/html/body/div[6]/section/section/div/section[1]/div/div[1]/div[2]/a[2]")
login_button.click()
iframe = driver.find_element_by_id("disneyid-iframe")
driver.switch_to.frame(iframe)
try:
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.XPATH, "/html/body/div[2]/div/div/section/section/form/section/div[1]/div/label/span[2]/input")))
element.send_keys("my username")
import time
time.sleep(100)
finally:
driver.quit()