I am trying to get the webdriver to click a button on the site random.org The button is a generator that generates a random integer between 1 and 100. It looks like this:
After inspecting the webpage, I found the corresponding element on the webpage looks something like this:
It is inside an iframe and someone suggested that I should first switch over to that iframe to locate the element, so I incorporated that in my code but I am constantly getting NoSuchElementException error. I have attached my code and the error measage below for your reference. I can't understand why it cannot locate the button element despite referencing the ID, which is supposed to unique in the entire document.
The code:
from selenium import webdriver
from selenium.webdriver.common.by import By
driver = webdriver.Edge()
driver.get("https://www.random.org/")
driver.implicitly_wait(15)
driver.switch_to.frame(driver.find_element(By.TAG_NAME, "iframe"))
button = driver.find_element(By.CSS_SELECTOR, "input[id='hnbzsqjufzxezy-button']")
button.click()
The error message:
Make sure that there are no more Iframes on the page. If there are a few an not only one do this:
iframes = driver.find_elements(By.CSS, 'iframe')
// try switching to each iframe:
driver.switch_to.frame(iframes[0])
driver.switch_to.frame(iframes[1])
You can't find the button because its name contain random letters. Every time you will refresh the page you can see that the name value will change. So, do this:
button = driver.findElement(By.CSS, 'input[type="button"][value="Generate"]')
button.click()
There are several issues with your code:
First you need to close the cookies banner
The locator of the button is wrong. It's id is dynamic.
You need to use WebDriverWait to wait for elements clickability.
The following code works:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(service=webdriver_service, options=options)
url = 'https://www.random.org/'
driver.get(url)
wait = WebDriverWait(driver, 10)
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button[onclick*='all']"))).click()
wait.until(EC.frame_to_be_available_and_switch_to_it((By.TAG_NAME, "iframe")))
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "input[id*='button']"))).click()
Related
I am trying to get some data from a website called : https://dexscreener.com/ethereum/0x1a89ae3ba4f9a97b10bac6a77061f00bb956858b
and i'm trying to get the element : /html/body/div[1]/div/main/div/div[2]/div/div[2]/div/div/div[1]/div[4]/div[2]/div[1]/div[1]/div[2]/span[2] which is basically a number on the webpage representing volume.
i used this code here:
driver.get('https://dexscreener.com/ethereum/' + str(tokenadress))
try:
fivemVolume = WebDriverWait(driver, delay).until(EC.presence_of_element_located(
(By.XPATH, '/html/body/div[1]/div/main/div/div[2]/div/div[2]/div/div/div[1]/div[4]/div[2]/div[1]/div[1]/div[2]/span[2]')))
except:
#more codee
I think its something to do with the webpage loading into some iframe as a default but when i added this code it didn't help:
driver.switch_to.default_content()
Your locator do not match any element on that page.
Elements you trying to access are inside iframe.
So, you need first to switch into the iframe.
The following code should work but I had problems running Selenium on that page since it is blocked by cloudflare:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(options=options, service=webdriver_service)
wait = WebDriverWait(driver, 30)
url = "https://dexscreener.com/ethereum/0x1a89ae3ba4f9a97b10bac6a77061f00bb956858b"
driver.get(url)
wait.until(EC.frame_to_be_available_and_switch_to_it((By.CSS_SELECTOR, "iframe[id*='tradingview']")))
value = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "[data-name='legend-source-item'] [class='valueItem-1WIwNaDF'] .valueValue-1WIwNaDF"))).text
print(value)
I have a python script, It look like this.
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.select import Select
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from os import path
import time
# Tried this code
chrome_options = webdriver.ChromeOptions()
prefs = {"profile.default_content_setting_values.notifications" : 2}
chrome_options.add_experimental_option("prefs",prefs)
browser = webdriver.Chrome(ChromeDriverManager().install(), chrome_options=chrome_options)
links = ["https://www.henleyglobal.com/", "https://markets.ft.com/data"]
for link in links:
browser.get(link)
#WebDriverWait(browser, 20).until(EC.url_changes(link))
#How do I disable/Ignore/remove/escape this "Accept all cookie" popup and then access the website to scrape data?
browser.quit()
So each website in the links array displays an "Accept all cookie" popup after navigating to the site. check the below image.
I have tried many ways nothing works, Check the one after imports
How do I exit/pass/escape this popup and then access the website to scrape data?
If you open your page in a new browser you'll note the page fully loads, then, a moment later your popup appears. The default wait strategy in selenium is just that the page is loaded.
One way to handle this is to simply inspect the page and find the xpath of the popup window. The below code should work for that.
browser.manage().timeouts().implicitlyWait(30, TimeUnit.SECONDS)
if link == 'https://www.henleyglobal.com/':
browser.findElement(By.XPATH("/html/body/div[7]/div/div/div/div[2]/div/div[2]/button[2]")).click()
else:
browser.findElement(By.XPATH("/html/body/div[4]/div/div/div[2]/div[2]/a")).click()
The code is waiting until the element of the pop-up is clickable and then clicking it.
For unknown sites you could try:
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--disable-notifications")
webdriver.Chrome(os.path.join(path, 'chromedriver'), chrome_options=chrome_options)
generally, you can not use some universal locator that will match the "Accept cookies" buttons for each and every web site in the world.
Even here, you have 2 different sites and the elements you need to click are totally different on these sites.
For https://www.henleyglobal.com/ site the correct locator may be something like this CSS Selector .confirmation button.primary-btn while for https://markets.ft.com/data site I'd advise to use CSS Selector .o-cookie-message__actions a.o-cookie-message__button.
These 2 elements are totally different: the first one is button while the second is a, they have totally different class names and all other attributes.
You may thing about the Accept text. It seems to be common, so you could use this XPath //*[contains(text(),'Accept')] but even this will not work since on the first page it matches 2 elements while the accept cookies element is the second between them...
So, there is no General locators, you will have to define separate locators for each page.
Again, for https://www.henleyglobal.com/ I would prefer
driver.find_element(By.CSS_SELECTOR, ".confirmation button.primary-btn").click()
While for the second page https://markets.ft.com/data I would prefer this
driver.find_element(By.CSS_SELECTOR, ".o-cookie-message__actions a.o-cookie-message__button").click()
Also, generally we always use WebDriverWait expected_conditions explicit waits, so the code will be as following:
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
wait = WebDriverWait(driver, 10)
# for the first page
wait.until(EC.element_to_be_clickable((By.XPATH, ".confirmation button.primary-btn"))).click()
# for the second page
wait.until(EC.element_to_be_clickable((By.XPATH, ".o-cookie-message__actions a.o-cookie-message__button"))).click()
I have tried to scrap info from that site - specifically, from a table. Every time I occur, info that elements doesn't exist.
https://polygonscan.com/token/0x64a795562b02830ea4e43992e761c96d208fc58d
I try to add time.slep(5) to my code or scrolling down function to load all element - ineffective.
Do you have any advice for me?
EDIT
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
# Options
chrome_options = Options()
chrome_options.add_argument("--headless")
# Set drive
chrome_driver_path = r"C:\Users\kacpe\OneDrive\Pulpit\Python\Projekty\chromedriver.exe"
driver = webdriver.Chrome(chrome_driver_path, options=chrome_options)
driver.get("https://polygonscan.com/token/0x64a795562b02830ea4e43992e761c96d208fc58d")
try:
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.XPATH, "//table/tbody/tr[0]")))
print(element)
except TimeoutException as e:
print(e)
I added code in regard to your request. So my main goal is to scrap content from the table at this site. I add Explicit Waits to my code and still I can't select anything from that table - it's looking like the script doesn't see anything from that area.
One way to try solve it, its using the Xpath of the element or the relative position of the same, to make Selenium, get allways the same "line" of position to return the "value" of the information that you are searching.
Ex1:find_element(:xpath,"//*[#id="wmd-input"]")#in that case it's the input of this check box.
If it doesnt work, try this one.
Ex2: browser.implicitly_wait(30) #makes a timer to load all the informations from the web to your machine.
I am trying to select a dropdown option in the form using Selenium webdriver in Python. The XPATH is correct, I also verified it is going to the right dropdown option but in the end it is not selecting it.
I have tried similar code for another website that has a dropdown. But it's not working for this particular website.
Can someone please help out with this?
from selenium import webdriver
driver = webdriver.Chrome("C:\\Users\\xxx\\Downloads\\chromedriver_win32\\chromedriver.exe")
driver.get("https://www.cersai.org.in/CERSAI/dbtrsrch.prg")
elem = driver.find_element_by_xpath("//select[#id='borrowerType']")
all_options = elem.find_elements_by_tag_name("option")
for option in all_options:
if option.get_attribute("value") == "IND":
option.click()
break
You should add a wait before accessing the dropdown element to make it loaded.
Also, this is a Select element, you can treat it in a special way as below:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Chrome("C:\\Users\\xxx\\Downloads\\chromedriver_win32\\chromedriver.exe")
wait = WebDriverWait(driver, 20)
driver.get("https://www.cersai.org.in/CERSAI/dbtrsrch.prg")
wait.until(EC.visibility_of_element_located((By.XPATH, "//select[#id='borrowerType']")))
select = Select(driver.find_element_by_xpath("//select[#id='borrowerType']"))
# select by visible text
select.select_by_value('IND')
It's strange that Select class did not work. It needs a JavaScript call.
driver = webdriver.Chrome(driver_path)
driver.maximize_window()
driver.implicitly_wait(50)
driver.get("https://www.cersai.org.in/CERSAI/dbtrsrch.prg")
driver.execute_script("return document.getElementById('borrowerType').selectedIndex = '2'")
I'm trying to get to the results page of a search but I have to first click on the dropdown option to complete the search. When I do this manually, the dropdown hides if I do not click on it right as it appears, when I code it, I get a the following error:
ElementNotInteractableException: Message: Element <div id="_esgratingsprofile_autocomplete-results-container" class="autocomplete-results-container msci-ac-search-data-dropdown"> could not be scrolled into view
This is my code so far, you can visit the url and see how it is yourself as well:
from selenium.webdriver import Firefox
from selenium.webdriver.support.ui import Select
from selenium.webdriver.firefox.options import Options
opts = Options()
opts.set_headless()
assert opts.headless
browser = Firefox(options=opts)
browser.get('https://www.msci.com/esg-ratings')
search_form = browser.find_element_by_id('_esgratingsprofile_keywords')
search_form.send_keys('MSFT')
browser.find_element_by_xpath("//div[#id='_esgratingsprofile_autocomplete-results-container']/ul[#id='ui-id-1']/li[#class='msci-ac-search-section-title ui-menu-item']").click()
I looked through many other answers but they didnt seem to deal with the case where the dropdown was not a directly clickable element or where it hides if you dont click on it right away. Any help is appreciated.
Try the below code, This code is working for me. Let me know if it shows any error.
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver import ActionChains
from selenium.webdriver.common.by import By
driver = webdriver.Chrome()
driver.maximize_window()
wait = WebDriverWait(driver, 5)
action = ActionChains(driver)
driver.get("https://www.msci.com/esg-ratings")
Drop_Down = driver.find_element_by_xpath('//*[#id="_esgratingsprofile_keywords"]')
Drop_Down.send_keys("MSFT")
# Select the First Result from the search.
Result = wait.until(
EC.presence_of_element_located((By.XPATH, "//div[contains(#class,'autocomplete-results-container')]/ul/li[1]")))
action.move_to_element(Result).click().perform()