I am having some trouble trying to automate some web inputs, but first i need to click some buttons and i cannot do it. I've tried a lot of stuff but i cannot complete it :'(
webpage: https://vacunacovid.catsalut.gencat.cat/
I cannot go past the image the code i have:
rom selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import time
# browser config
options = webdriver.ChromeOptions()
options.add_argument('--start-maximized')
options.add_argument('--disable-extensions')
driver_path= "/C:/chromedriver_linux64/chromedriver" # este es el driver del navegador https://chromedriver.chromium.org/
driver = webdriver.Chrome(driver_path,chrome_options=options)
#starting screen in optimal position
driver.set_window_position(2000,0)
driver.maximize_window()
time.sleep(1)
#getting website
driver.get("https://vacunacovid.catsalut.gencat.cat/")
#go to the element if its clickable
WebDriverWait(driver, 5)\
.until(EC.element_to_be_clickable((By.XPATH, BLABLABLA)))\
.click()
it doesn't find anything throwugh this #shadow (open) , how can i do it?
https://i.stack.imgur.com/it2nQ.png
It is under Shadow-dom #shadow-root (open) So you have not mentioned exactly which button you want to click, so I'm clicking on the first button Demana o modifica cita
#adding some wait for application to load properly
sleep(5)
You just take the JS path of the desire element as below and return the element for that
press F12->Element Tab -> right click(on the element)->copy JS path
javascript = 'return document.querySelector("body > vaccinapp-app").shadowRoot.querySelector("#pages > vaccinapp-shell").shadowRoot.querySelector("#main-shell-content > appointment-shell").shadowRoot.querySelector("#appointment-shell-content > appointment-onboarding").shadowRoot.querySelector("#dismiss-btn").shadowRoot.querySelector("#button")'
By using execute_script will access the element under shadow-root (open)
element = driver.execute_script(javascript)
element.click()
code
options = webdriver.ChromeOptions()
options.add_argument('--start-maximized')
options.add_argument('--disable-extensions')
driver_path= "/C:/chromedriver_linux64/chromedriver"
driver = webdriver.Chrome(driver_path,chrome_options=options)
#starting screen in optimal position
driver.set_window_position(2000,0)
driver.maximize_window()
time.sleep(1)
#getting website
driver.get("https://vacunacovid.catsalut.gencat.cat/")
sleep(5)
javascript = 'return document.querySelector("body > vaccinapp-app").shadowRoot.querySelector("#pages > vaccinapp-shell").shadowRoot.querySelector("#main-shell-content > appointment-shell").shadowRoot.querySelector("#appointment-shell-content > appointment-onboarding").shadowRoot.querySelector("#dismiss-btn").shadowRoot.querySelector("#button")'
element = driver.execute_script(javascript)
element.click()
For reference check here
Related
I'm trying to crawl a site and am running into a google ad. I think I've found the iframe of it but I can't find the element to click to remove the ad. I've spent about 7 hours now and think this is over my head. Help v much appreciated.
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
chrome_options = Options()
chrome_options.add_argument("--incognito")
chrome_options.add_argument("--window-size=1920x1080")
# chrome_options.add_argument("--headless")
driver = webdriver.Chrome(chrome_options=chrome_options, executable_path ='C:\/Users\/gblac\/OneDrive\/Desktop\/Chromedriver.exe')
url = 'https://free-mp3-download.net/'
driver.get(url)
WebDriverWait(driver, 4)
search = driver.find_element(By.ID,'q')
search.send_keys('testing songs')
search.click()
button = driver.find_element(By.ID,'snd')
button.click()
WebDriverWait(driver,20).until(EC.visibility_of_element_located((By.CLASS_NAME,'container'))).click()
WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.ID,"results_t")));
results = driver.find_element(By.ID,'results_t').find_elements(By.TAG_NAME,'tr')
results[0].find_element(By.TAG_NAME,'a').click()
# The code to remove the ad would go here
# driver.find_elements(By.CSS_SELECTOR,'[text()="Close"]').click()
Add the below code block in your code - before searching any text:
time.sleep(1)
driver.execute_script("""
const elements = document.getElementsByClassName("google-auto-placed");
while (elements.length > 0) elements[0].remove();
""")
time.sleep(1)
driver.execute_script("""
const elements = document.getElementsByClassName("adsbygoogle adsbygoogle-noablate");
while (elements.length > 0) elements[0].remove();
""")
time.sleep(1)
driver.find_element(By.ID,"q").send_keys("tamil songs")
driver.find_element(By.ID,"snd").click()
It will close the 2 ad blocks in that page, but if you refresh or move forward and backward, the ads will display again, then you have to remove those ad blocks again using the above code, please add the code for that condition.
WebDriverWait(driver,20).until(EC.visibility_of_element_located((By.CLASS_NAME,'container'))).click()
WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.ID,"results_t")))
results = driver.find_element(By.ID,'results_t').find_elements(By.TAG_NAME,'tr')
results[0].find_element(By.TAG_NAME,'a').click()
time.sleep(2)
driver.find_element(By.XPATH, ".//button[contains(text(),'Download')]").click()
driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
time.sleep(1)
# handling captcha
iframe_captcha = driver.find_element(By.XPATH,".//iframe[#title='reCAPTCHA']")
driver.switch_to.frame(iframe_captcha)
time.sleep(1)
driver.find_element(By.CSS_SELECTOR, ".recaptcha-checkbox-border").click()
time.sleep(2)
driver.switch_to.default_content()
driver.find_element(By.XPATH, ".//button[contains(text(),'Download')]").click()
I am trying to press the accept button in a cookies popup in the website https://www.immobilienscout24.de/
Snapshot:
I understand that this requires
driver.execute_script("""return document.querySelector('#usercentrics-root')""")
But I can't trickle down the path to the accept button in order to click it. Can anyone provide some help?
This is one way (tested & working) you can click that button: please observe the imports, as well as the code after defining the browser/driver:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
chrome_options = Options()
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument('disable-notifications')
import time as t
webdriver_service = Service("chromedriver/chromedriver") ## path to where you saved chromedriver binary
browser = webdriver.Chrome(service=webdriver_service, options=chrome_options)
actions = ActionChains(browser)
url = 'https://www.immobilienscout24.at/regional/wien/wien/wohnung-kaufen'
browser.get(url)
page_title = WebDriverWait(browser, 3).until(EC.presence_of_element_located((By.CSS_SELECTOR, "a[title='Zur Homepage']")))
actions.move_to_element(page_title).perform()
parent_div = WebDriverWait(browser, 20000).until(EC.presence_of_element_located((By.ID, "usercentrics-root")))
shadowRoot = browser.execute_script("return arguments[0].shadowRoot", parent_div)
try:
button = WebDriverWait(shadowRoot, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button[data-testid='uc-accept-all-button']")))
button.click()
print('clicked')
except Exception as e:
print(e)
print('no click button')
That page is reacting to user's behavior, and it will only fully load the page once it detects mouse movements, hence the ActionChains() part of the code. After that, we drill down into the shadow root element, we locate the button (using Waits, to make sure it's clickable), and then we click it.
Selenium documentation can be found at https://www.selenium.dev/documentation/
The element Alle akzeptieren within the website is located within a #shadow-root (open).
Solution
To click on the element Alle akzeptieren you have to use shadowRoot.querySelector() and you can use the following Locator Strategy:
Code Block:
driver.execute("get", {'url': 'https://www.immobilienscout24.de/'})
time.sleep(10)
item = driver.execute_script('''return document.querySelector('div#usercentrics-root').shadowRoot.querySelector('button[data-testid="uc-accept-all-button"]')''')
item.click()
Here is the link to the site I am currently viewing: https://messari.io/tool/fb8d86ca-d3cf-4568-8d48-1a052c95364e. Scroll to the bottom of the page and click "View More".
I am trying to figure out how to click the x button but what I have tried hasn't worked. I get a "no such element: Unable to locate element error.
I have tried all three of these:
driver.find_element(By.CLASS_NAME, "button").click()
driver.find_element_by_xpath("//button[contains(#data-testid='CloseIcon')]").click()
driver.find_element_by_tag_name("svg").click()
Check the Xpath of the load more button //*[#id="root"]/div[2]/div/div[2]/div[2]/div[3]/button -
And then check the full Xpath of the X button that will close the pop-up window
full Xpath of the X button - '/html/body/div[2]/div[3]/div/h2/button'
working code -
import time
from selenium import webdriver
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from webdriver_manager.chrome import ChromeDriverManager
options = webdriver.ChromeOptions()
# options.add_argument("--headless")
options.add_argument("--no-sandbox")
options.add_argument("--disable-gpu")
options.add_argument("--window-size=1920x1080")
options.add_argument("--disable-extensions")
chrome_driver = webdriver.Chrome(
service=Service(ChromeDriverManager().install()),
options=options
)
def messari_scraper():
URL = "https://messari.io/tool/fb8d86ca-d3cf-4568-8d48-1a052c95364e"
with chrome_driver as driver:
driver.implicitly_wait(15) # wait max 15 sec for any element to find
driver.get(URL)
time.sleep(3)
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);") # scroll to the end of the page
# click the button
driver.find_element(By.XPATH, '//*[#id="root"]/div[2]/div/div[2]/div[2]/div[3]/button').click()
time.sleep(3)
# get the full Xpath of the close `x` button of the pop-up
driver.find_element(By.XPATH, '/html/body/div[2]/div[3]/div/h2/button').click()
# pop up window closed
time.sleep(5)
# do your tasks here....
messari_scraper()
I am new to selenium and web development. I am working on a project to take screenshots from the websites from web.archive.org.
Here is a link to the sample webpage. I am trying to click on the close button (on the top right of this page) before capturing the screenshots. I am not sure what kind of element is the close button and I was unsuccessful in my attempts.
Here is the element I am attempting to click from selenium:
Here is the corresponding HTML
<a id="wm-tb-close" href="#close" style="top:-2px;" title="Close the toolbar"><span class="iconochive-remove-circle" style="color:#888888;font-size:240%;"></span></a>
Here is my code:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
options = Options()
options.add_argument('--headless')
options.add_argument('--no-sandbox')
options.add_argument('--start-maximized')
options.add_argument('--disable-dev-shm-usage')
options.binary_location = "<path-to-local-dir>/google-chrome/opt/google/chrome/google-chrome"
driver = webdriver.Chrome(options=options)
driver.get('https://web.archive.org/web/20220315011343/https://stackoverflow.com/')
# My attempts at closing the wayback toolbar:
# driver.find_element(By.LINK_TEXT, 'close').click() # Attempt 1
# driver.find_element(By.ID, 'wm-tb-close').click() # Attempt 2
# driver.find_element_by_xpath("a[#title='Close the toolbar']").click() # Attempt 3
# Capture full webpage screenshot (with scrolling)
original_size = driver.get_window_size()
# required_width = driver.execute_script('return document.body.parentNode.scrollWidth')
required_height = driver.execute_script('return document.body.parentNode.scrollHeight')
driver.set_window_size(1920, max(required_height, 1080))
driver.find_element(By.TAG_NAME, 'body').screenshot('webpage_screenshot.png') # avoids scrollbar
driver.set_window_size(original_size['width'], original_size['height']) # reset to defaults
driver.quit()
When I make an attempt to click on the close button using the three techniques (shown in the code), I receive the following error:
selenium.common.exceptions.NoSuchElementException: Message: no such
element: Unable to locate element
The element you are trying to click is inside the SHADOW-ROOT, to access such elements you need to use some special techniques.
Also you need to use Expected Conditions explicit waits to let the elements loaded before accessing them.
This should work:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument('--headless')
options.add_argument('--no-sandbox')
options.add_argument('--start-maximized')
options.add_argument('--disable-dev-shm-usage')
options.binary_location = "<path-to-local-dir>/google-chrome/opt/google/chrome/google-chrome"
driver = webdriver.Chrome(options=options)
wait = WebDriverWait(driver, 20)
driver.get('https://web.archive.org/web/20220315011343/https://stackoverflow.com/')
#locate the shadow root element
root_element = wait.until(EC.presence_of_element_located((By.ID, "wm-ipp-base")))
shadow_root = driver.execute_script('return arguments[0].shadowRoot', root_element)
#access the close button inside the shadow root
shadow_root.find_element_by_css_selector("span.iconochive-remove-circle").click()
I am trying to click on the button "View all details" to expand the details on a restaurant from OpenTable but I keep getting a no element exception.
from selenium import webdriver
driver = webdriver.Chrome(
'/Library/Python/2.7/site-packages/chromedriver')
url = "https://www.opentable.com/chicago-illinois-restaurant-listings"
driver.get(url)
element = driver.find_element_by_xpath(
'//*[#id="search_results"]/div[2]/div[1]/div/div[2]/div[1]/a')
element.click()
driver.find_element_by_css_selector(
'#overview-section > div:nth-child(4) > div.f9f46391 > button').click()
driver.quit()
Each result link has target='_blank' attribute. That means that if to click the link details page will be opened in new tab. To handle elements on new tab you should switch to it:
driver.get(url)
current = driver.current_window_handle
driver.find_element_by_css_selector('a.rest-row-name').click()
driver.switch_to.window([tab for tab in driver.window_handles if tab != current][0])
Note that you should also wait for button to became clickable:
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait as wait
from selenium.webdriver.support import expected_conditions as EC
wait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//button[.="View all details"]'))).click()