So I am trying to scrape a website, and I want to click an element, go to the page that opens from the click, find another element and click that one. The first click seems to work, no errors, but the next page doesn't open, thus I get an error. Here is a screenshot of what I want to click on the fist page: https://prnt.sc/10l8xa4. Clicking that should redirect to the second page. The problem seems to be that the driver clicks the element but nothing happens:
import sys, csv, os
from selenium import webdriver # Selenium 3.141.0
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup
from datetime import datetime
from time import sleep
class Scraper(object):
''' A lot of the messy code is just playing with the tags from the page'''
def __init__(self, link):
self.link = link
self.driver = self.configure_driver() # The simulated browser
# Configuring the browser simulator, named driver, that will get all the information
def configure_driver(self):
# Add additional Options to the webdriver
chrome_options = Options()
# add the argument and make the browser Headless. It will work smoother& faster but it will miss the first category
# chrome_options.add_argument("--headless")
driver = webdriver.Chrome(options = chrome_options)
return driver
def click_element(self, selector): # Clicks the provided element from the page, even if not visible
element = WebDriverWait(self.driver, 20).until(
EC.presence_of_element_located((By.CSS_SELECTOR, selector)))
ActionChains(self.driver).move_to_element(element).click(element).perform()
if __name__ == '__main__':
product_link = 'https://www.action.com/nl-nl/click-and-collect-producten/' # An example of a product
app = Scraper(product_link)
with app.driver:
app.driver.get(product_link)
app.click_element('a.content-card.has-text.card-theme--light.card-size--s.card-align--bottom-left') # This gets clicked and should open new page, but it doesn't
sleep(10)
app.click_element('a.product-card__link') # This throws a Timeout, because the element can't be found, which is obvoius because the second page(which has this element) didn't open
sleep(20)
Try use like that:
with app.driver:
app.driver.get(product_link)
sleep(2)
app.click_element('li.has-submenu')
sleep(2)
app.click_element(
'div.grid-item.grid-item--content') # This gets clicked and should open new page, but it doesn't
sleep(2)
you should add step of openning pop-up and then click on your aimed button
Code with required argument but, by xpath:
def click_element(self, selector, by=By.CSS_SELECTOR): # Clicks the provided element from the page, even if not visible
element = WebDriverWait(self.driver, 20).until(
EC.presence_of_element_located((by, selector)))
ActionChains(self.driver).move_to_element(element).click(element).perform()
if __name__ == '__main__':
product_link = 'https://www.action.com/nl-nl/click-and-collect-producten/' # An example of a product
app = Scraper(product_link)
with app.driver:
app.driver.get(product_link)
sleep(2)
app.click_element("//section[#class='grid']/div[#class='grid-item grid-item--content'][1]", By.XPATH)
sleep(2)
Related
Selenium does see displayed element visible on the page on the second iteration.
I click on a link, and a box within a website appears. I need to close that box.
This action will be performed 1000+ times. On the first iteration, Selenium opens the link and closes the box. On the second iteration, Selenium opens the link, and cannot close the box. At this point, it gives error message:
Exception has occurred: ElementNotInteractableException Message: element not interactable (Session info: chrome=105.0.5195.102)
My code + HTML of relevant element below.
from selenium import webdriver
from selenium.webdriver.common.by import By
import time
options = webdriver.ChromeOptions()
options.add_argument("start-maximized")
options.add_argument("disable-infobars")
options.add_argument("--disable-extensions")
driver = webdriver.Chrome(chrome_options=options, executable_path=r"D:\SeleniumDriver\chromedriver.exe")
driver.get('https://sprawozdaniaopp.niw.gov.pl/')
find_button = driver.find_element("id", "btnsearch")
find_button.click()
interesting_links = driver.find_elements(By.CLASS_NAME, "dialog")
for i in range(len(interesting_links)):
interesting_links[i].click()
time.sleep(10) # I tried 60 seconds, no change
#
# HERE I WOULD DO MY THINGS
#
close_box = driver.find_element(By.CLASS_NAME, "ui-dialog-titlebar-close")
print(close_box.is_displayed())
close_box.click() # Here is where the program crushes on the 2nd iteration
if i == 4: # Stop the program after 5 iterations
break
HTML code of the relevant element:
<span class="ui-icon ui-icon-closethick">close</span>
I tried to locate the element that closes the box by CSS SELECTOR AND XPATH.
The CSS SELECTOR of the X/close button is the same every time, but
only the first time Selenium will see the X button displayed.
THE XPATH is strange. On the first opening of the link, X/close button will have path:
/html/body/div[6]/div[1]/a
However, if you open the next link, path will look this:
/html/body/div[8]/div[1]/a
Let me know what you think of that :-)
This is one way to achieve your goal:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time as t
chrome_options = Options()
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument('disable-notifications')
chrome_options.add_argument("window-size=1280,720")
webdriver_service = Service("chromedriver/chromedriver") ## path to where you saved chromedriver binary
browser = webdriver.Chrome(service=webdriver_service, options=chrome_options)
wait = WebDriverWait(browser, 20)
url = 'https://sprawozdaniaopp.niw.gov.pl/'
browser.get(url)
wait.until(EC.element_to_be_clickable((By.ID, "btnsearch"))).click()
links = wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "a[class='dialog']")))
counter = 0
for link in links[:5]:
link.click()
print('clicked link', link.text)
### do your stuff ###
t.sleep(1)
wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, 'span[class="ui-icon ui-icon-closethick"]')))[counter].click()
print('closed the popup')
counter = counter+1
This will print out in terminal:
clicked link STOWARZYSZENIE POMOCY DZIECIOM Z PORAŻENIEM MÓZGOWYM "JASNY CEL"
closed the popup
clicked link FUNDACJA NA RZECZ POMOCY DZIECIOM Z GRODZIEŃSZCZYZNY
closed the popup
clicked link FUNDACJA "ADAMA"
closed the popup
clicked link KUJAWSKO-POMORSKI ZWIĄZEK LEKKIEJ ATLETYKI
closed the popup
clicked link "RYBNICKI KLUB PIŁKARSKI - SZKÓŁKA PIŁKARSKA ROW W RYBNIKU"
closed the popup
Every time you click on a link, a new popup is created. When you close it, that popup will not disappear, but it will stay hidden. So when you click on a new link and then you want to close the new popup, you need to select the new (nth) close button. This should also apply to popup elements, so make sure you account for it. I stopped after the 5th link, of course you will need to remove the slicing to handle all links present in page.
Selenium setup above is chromedriver on linux - you just have to observe the imports, and the code after defining the browser(driver).
Selenium documentation can be found at https://www.selenium.dev/documentation/
I have the following code as part of a data download function in selenium / chrome driver
driver.find_element_by_class_name("mt-n1").click()
driver.implicitly_wait(5)
When I 'step through' the code it works fine, but in normal execution it fails to action this link on the web page, therefore does not open the next webpage as required by the function
I have inspected the web page and identified the xpath, so I amended the code as follows
driver.find_element_by_xpath("//div[#class = 'mt-n1 flex-auto']").click()
driver.implicitly_wait(5)
This produces the same result, i.e. only works if I step through the code one line at a time
Hopefully just something obvious I'm not seeing?
The full code is given below for ref.
# Download file from github function
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
workDir = r"/C:\Users\Oem\Desktop"
### Set the download directory
chromeOptions = webdriver.ChromeOptions()
chromeOptions.add_experimental_option("prefs", {"download.default_directory": workDir})
### Open a web-browser for automated control
driver = webdriver.Chrome(options=chromeOptions)
driver.get("https://github.com")
driver.fullscreen_window()
time.sleep(5)
assert "GitHub" in driver.title
elem = driver.find_element_by_class_name("header-search-input")
elem.clear()
elem.send_keys("acca-edx/python-practical-automation")
elem.send_keys(Keys.RETURN)
assert "No results found." not in driver.page_source
### Go to the first result.
# Find the results on the page using "mt-n1" and clicking on it.
# Wait for the page to complete.
driver.find_element_by_class_name("mt-n1").click()
driver.implicitly_wait(5)
driver.find_element_by_xpath("//a[#title='detection.xlsx']").click()
driver.implicitly_wait(5)
# Find the raw-url to download the file we are after.
driver.find_element_by_id('raw-url').click()
time.sleep(10)
driver.close()
if your code works while going step by step through, it might be that web page needs time to be rendered. try this, it will wait 50 seconds or until the element presence has been located.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import TimeoutException
driver = webdriver.Chrome(options=chromeOptions)
try:
element = WebDriverWait(driver, 50).until(EC.presence_of_element_located((By.CLASS_NAME, 'mt-n1')))
except TimeoutException as e:
# do something if element is not found
I'm trying to make a script that logs into my online grade book to look for any changes (new grades, etc). This is my code so far.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager
import time
def main():
options = Options()
# options.add_argument("--headless")
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)
driver.maximize_window()
# goes to the desired website
driver.get('https://portal.librus.pl/rodzina')
# searches for and clicks a button that drops down a menu in which link for login form is visible
button = driver.find_element(By.CLASS_NAME, 'btn.btn-third.btn-synergia-top.btn-navbar.dropdown-toggle')
button.click()
# searches and clicks login link
agree = driver.find_element(By.CLASS_NAME, 'zmdi.zmdi-account.dropdown-item__icon')
agree.click()
time.sleep(10)
driver.quit()
if __name__ == '__main__':
main()
And there is a problem, I cannot seem to find a way to make webdriver see what I see. What I mean is that I see the webpage like this and webdriver sees the same webpage like this also the source code is different. I've tried using undetected ChromeDriver with no success. This is my code using UC.
import undetected_chromedriver as uc
import time
from selenium.webdriver.common.by import By
def main():
driver = uc.Chrome()
driver.maximize_window()
# goes to the desired website
driver.get('https://portal.librus.pl/rodzina/home')
# searches for and clicks a button that drops down a menu in which link for login form is visible
button = driver.find_element(By.CLASS_NAME, 'btn.btn-third.btn-synergia-top.btn-navbar.dropdown-toggle')
button.click()
# searches and clicks login link
agree = driver.find_element(By.CLASS_NAME, 'zmdi.zmdi-account.dropdown-item__icon')
agree.click()
time.sleep(5)
driver.execute_script("window.print();")
if __name__ == '__main__':
main()
Has anyone had a similar problem and managed to solve it?
I'm new to Selenium, just started looking at it a few days ago.
Here's my program.
It just basically opens a website, nothing too much.
But it does not loads the window, just prints data:, to the window header, and a white background color pops up.
from selenium import webdriver
PATH = "/home/MyName/_DEV_/Selenium/chromedriver"
driver = webdriver.Chrome(PATH)
driver.get('google.com')
#driver.close()
What can be the problem? Thanks for the help in advance.
I understand being new. I have provided you a little "template" on what I use as a basis to help you get started. ( Note: I use this in my own custom classes; but, for demonstration purposes, I put it all into one file for you ).
MAIN PROGRAM - For Reference
from selenium import webdriver
from selenium.webdriver.chrome.webdriver import WebDriver as ChromeDriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait as DriverWait
from selenium.webdriver.support import expected_conditions as DriverConditions
def get_chrome_driver():
"""This sets up our Chrome Driver and returns it as an object"""
path_to_chrome = "F:\Selenium_Drivers\Windows_Chrome85_Driver\chromedriver.exe"
chrome_options = webdriver.ChromeOptions()
# Browser is displayed in a custom window size
chrome_options.add_argument("window-size=1500,1000")
return webdriver.Chrome(executable_path = path_to_chrome,
options = chrome_options)
def is_displayed(driver : ChromeDriver, xpath : str, timeout = 5):
"""Checks if our element displays on our page. If it does, return True. Otherwise, return False."""
try:
webElement = DriverWait(driver, timeout).until(
DriverConditions.presence_of_element_located(locator = (By.XPATH, xpath))
)
return True if webElement != None else False
except:
return False
# Gets our chrome driver and opens our site
chrome_driver = get_chrome_driver()
chrome_driver.get("https://www.google.com/")
result = is_displayed(chrome_driver, "//input[#title='Search']")
print(f'Does Search Textbox Display: {result}')
chrome_driver.quit()
chrome_driver.service.stop()
Solved:
First made sudo apt install chromium-chromedriver, than
from selenium import webdriver
driver = webdriver.Chrome()
driver.get(https://google.com')
I've written a script in python with selenium. The script is supposed to click on some links in a webpage. When I run my script, It does click on the first link and then throws an error stale element reference: element is not attached to the page document instead of chasing for the next link. I searched a lot for the last few hours to find any solution to get rid of this error but no luck.
I'm not interested in their data so any solution other than the perocess of clicking is not what I'm looking for. How can I click on the links until the last link?
This is my attempt so far:
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
def click_links(driver,url):
driver.get(url)
for olink in wait.until(EC.presence_of_all_elements_located((By.CLASS_NAME, "result-row__item-hover-visualizer"))):
olink.click()
time.sleep(3)
if __name__ == '__main__':
weblink = "https://www.hitta.se/s%C3%B6k?vad=Markiser+%26+Persienner"
driver = webdriver.Chrome()
wait = WebDriverWait(driver, 10)
try:
click_links(driver,weblink)
finally:
driver.quit()
You can try below code:
def click_links(driver,url):
driver.get(url)
links_len = len(wait.until(EC.presence_of_all_elements_located((By.CLASS_NAME, "result-row__item-hover-visualizer"))))
for index in range(links_len):
cookies_bar = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '[data-bind="visible: showCookieDialog"]')))
driver.execute_script("arguments[0].hidden='true';", cookies_bar)
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, 'button[data-track="click-show-more"]'))).click()
entry = wait.until(EC.presence_of_all_elements_located((By.CLASS_NAME, "result-row__item-hover-visualizer")))[index]
entry.click()
time.sleep(3)
driver.back()