I've written a script in python to click on some categories in a webpage. I could manage to click on the first two categories but got stuck when it comes to initiate the final click. I've given a link leading to the two images in I have marked where to click.
This is the first link where there is a sign (marked with pencil) to click on to enter the second portion.
This is the second link where I get stuck when I try to click on the names (I've marked those names with pencil)
This is the site link.
Script I've tried with so far:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Chrome()
wait = WebDriverWait(driver, 10)
driver.get("replace_with_above_link")
wait.until(EC.element_to_be_clickable((By.CLASS_NAME, "i4ewOd-pzNkMb-ornU0b-b0t70b-Bz112c"))).click()
post = wait.until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, "div[role='checkbox']")))[1]
post.click()
for item in wait.until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR,".HzV7m-pbTTYe-JNdkSc .suEOdc"))):
item.click()
driver.quit()
My intention is to click the names cyclically. Thanks in advance.
Try below code to click each item in list:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Chrome()
wait = WebDriverWait(driver, 10)
driver.get(URL)
wait.until(EC.element_to_be_clickable((By.CLASS_NAME, "i4ewOd-pzNkMb-ornU0b-b0t70b-Bz112c"))).click()
post = wait.until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, "div[role='checkbox']")))[1]
post.click()
for item in wait.until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR,".HzV7m-pbTTYe-JNdkSc .suEOdc")))[1:]:
item.click()
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, ".HzV7m-tJHJj-LgbsSe-Bz112c.qqvbed-a4fUwd-LgbsSe-Bz112c"))).click()
wait.until(EC.invisibility_of_element_located((By.CSS_SELECTOR, ".qqvbed-p83tee")))
driver.quit()
Related
I'm trying to use selenium to open up this webpage (https://app.powerbi.com/view?r=eyJrIjoiMzE2ZDIyN2YtODY1Yy00ZGY0LWE4YTktNDcxOTcwYWQyMjM5IiwidCI6IjcyMmVhMGJlLTNlMWMtNGIxMS1hZDZmLTk0MDFkNjg1NmUyNCJ9) and click on the Tram icon to navigate to the web page I want to scrape.
This is what I've tried up to now
from selenium import webdriver
driver=webdriver.Chrome()
driver.get("https://app.powerbi.com/view?r=eyJrIjoiMzE2ZDIyN2YtODY1Yy00ZGY0LWE4YTktNDcxOTcwYWQyMjM5IiwidCI6IjcyMmVhMGJlLTNlMWMtNGIxMS1hZDZmLTk0MDFkNjg1NmUyNCJ9")
driver.maximize_window()
x=driver.find_element("class name", "imageBackground")
print(x)
#driver.find_element("class name", "imageBackground").click()
However, the element returns none. I'm not sure how to find the element and click the div to navigate to the next page as well.
I've updated your code.
I added a WebDriverWait that pauses until the tram element is clickable, then it clicks on the Tram element and goes to the next page.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
if __name__ == "__main__":
driver=webdriver.Chrome()
driver.get("https://app.powerbi.com/view?r=eyJrIjoiMzE2ZDIyN2YtODY1Yy00ZGY0LWE4YTktNDcxOTcwYWQyMjM5IiwidCI6IjcyMmVhMGJlLTNlMWMtNGIxMS1hZDZmLTk0MDFkNjg1NmUyNCJ9")
driver.maximize_window()
wait = WebDriverWait(driver, 30)
tramElementXpath = '//*[#aria-label="Tram Bookmark . Click here to follow link"]'
x = wait.until(EC.element_to_be_clickable((By.XPATH, tramElementXpath)))
x.click()
I'm trying to login to a webpage using Selenium but when I enter the relevant email address and try to click continue I get the no such element exception.
driver = webdriver.Safari()
driver.get('https://manage.statuspage.io/login')
email = driver.find_element_by_id('email')
email.clear()
email.send_keys('XXX')
driver.find_element_by_id("login-btn").click()
driver.find_element_by_id("login-submit").click()
The error occurs on the last line.
As you can see I'm searching for this continue button via id 'login-submit' but it says that it doesn't exist.
Any help would be great!
After clicking
driver.find_element_by_id("login-btn").click()
It takes you to another page where the login-submit element is presented.
But you trying to click this element immediately after clicking on element on previous page while the new page is still not loaded.
You should add a wait to wait for that element on the second page to appear before accessing it.
This should work:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Safari()
wait = WebDriverWait(driver, 20)
driver.get('https://manage.statuspage.io/login')
email = wait.until(EC.visibility_of_element_located((By.ID, "email")))
email.clear()
email.send_keys('XXX')
wait.until(EC.visibility_of_element_located((By.ID, "login-btn"))).click()
wait.until(EC.visibility_of_element_located((By.ID, "login-submit"))).click()
You need to wait for the element/locator to appear on the webpage.
from selenium.webdriver.support import expected_conditions as EC
driver.find_element_by_id("login-btn").click()
login_button = WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.ID, 'login-submit')))
login_button.click()
You can use explicitWait like below to click on the desired button
driver.find_element_by_id("email").send_keys("email")
OR
email = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.ID, "email")))
email.send_keys('email')
btnLogin = WebDriverWait(driver,10).until(EC.element_to_be_clickable((By.ID,"login-btn")))
btnLogin.click()
import
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
What happened sometime when you send request from a script it returns you page with different ids or class names. After driver.get('https://manage.statuspage.io/login') line save the webpage as .html file and see the id or class name there.
I am using selenium to try to scrape data from a website (https://www.mergentarchives.com/), and I am attempting to get the innerText from this element:
<div class="x-paging-info" id="ext-gen200">Displaying reports 1 - 15 of 15</div>
This is my code so far:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
driver = webdriver.Firefox()
driver.maximize_window()
search_url = 'https://www.mergentarchives.com/search.php'
driver.get(search_url)
assert 'Mergent' in driver.title
company_name_input = '//*[#id="ext-comp-1009"]'
search_button = '//*[#id="ext-gen287"]'
driver.implicitly_wait(10)
driver.find_element_by_xpath(company_name_input).send_keys('3com corp')
driver.find_element_by_xpath(search_button).click()
driver.implicitly_wait(20)
print(driver.find_element_by_css_selector('#ext-gen200').text)
basically I am just filling out a search form, which works, and its taking me to a search results page, where the number of results is listed in a div element. When I attempt to print the text of this element, I simply get a blank space, there is nothing written and no error.
[Finished in 21.1s]
What am I doing wrong?
I think you may need explicit Wait :
wait = WebDriverWait(driver, 10)
info = wait.until(EC.visibility_of_element_located((By.XPATH, "//div[#class = 'x-paging-info' and #id='ext-gen200']"))).get_attribute('innerHTML')
print(info)
Imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
You may need to put a condition by verifying if search results loaded or not and once its loaded you can use below code
print(driver.find_element_by_id('ext-gen200').text)
I've written a script in Python in association with selenium to keep clicking on MORE button to load more items until there are no new items left to load from a webpage. However, my below script can click once on that MORE button available on the bottom of that page.
Link to that site
This is my try so far:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
link = "https://angel.co/companies?company_types[]=Startup&company_types[]=Private+Company&company_types[]=Mobile+App&locations[]=1688-United+States"
driver = webdriver.Chrome()
wait = WebDriverWait(driver, 10)
driver.get(link)
while True:
for elems in wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR,".results .name a.startup-link"))):
print(elems.get_attribute("href"))
try:
loadmore = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR,"[class='more']")))
driver.execute_script("arguments[0].scrollIntoView();", loadmore)
loadmore.click()
except Exception:break
driver.quit()
How can I keep clicking on that MORE button until there are no such button left to click and parse the links as I've already tried using for loop.
I've managed to solve the problem pursuing sir Andersson's logic within my exising script. This is what the modified script look like.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
link = "https://angel.co/companies?company_types[]=Startup&company_types[]=Private+Company&company_types[]=Mobile+App&locations[]=1688-United+States"
driver = webdriver.Chrome()
wait = WebDriverWait(driver, 10)
driver.get(link)
while True:
try:
loadmore = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR,"[class='more']")))
driver.execute_script("arguments[0].click();", loadmore)
wait.until(EC.staleness_of(loadmore))
except Exception:break
for elems in wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR,".results .name a.startup-link"))):
print(elems.get_attribute("href"))
driver.quit()
why not just?
while (driver.FindElements(By.ClassName("more")).Count > 0)
{
driver.FindElement(By.ClassName("more")).Click();
//Some delay to wait lazyload to complete
}
c# example. pretty sure that it can be done with python as well
This is the webpage -- link. I am going to crawl (In Persian). I have a problem when I am going to click on the next page button. The XPath is:
nextpage = '//*[#id="ctl00_ContentPlaceHolder1_ASPxSplitter1_CallbackPaneldgd_dgd_DXPagerBottom"]/a[1]/img'
page = driver.find_element_by_xpath(nextpage)
page.click()
After the page.click() I got the following error:
selenium.common.exceptions.ElementNotVisibleException: Message: element not visible
Some answers say there might be a duplicate XPath, but I could not find such a thing in the source of the webpage.
The complete code:
import selenium
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from selenium.webdriver import ActionChains
driver = webdriver.Chrome(executable_path='./chromedriver')
url = 'http://hmi.mrud.ir/sabaa/SABAA/Home/Default.aspx?strTownship=0101&g=false'
driver.get(url)
time.sleep(10)
nextpage = '//*[#id="ctl00_ContentPlaceHolder1_ASPxSplitter1_CallbackPaneldgd_dgd_DXPagerBottom"]/a[1]/img'
page = driver.find_element_by_xpath(nextpage)
page.click()
Thanks.
Use the following before clicking on the element:
driver.execute_script("arguments[0].scrollIntoView(true);", page)
This is scrolling to the element.
Hope it helps you!
As per the Website if you want to click on the Next Page Button you have to induce WebDriverWait for the WebElement to be clickable following either of the code block as follows :
nextpage = WebDriverWait(driver, 10).until(EC.element_to_be_clickable(By.XPATH,"//div[#id='ctl00_ContentPlaceHolder1_ASPxSplitter1_CallbackPaneldgd_dgd_DXPagerBottom']/a[#class='dxp-lead dxp-button dxp-bi']/img[#class='dxWeb_pPrev_Aqua']"))
nextpage.click()
Or
nextpage = WebDriverWait(driver, 10).until(lambda x: x.find_element_by_xpath("//div[#id='ctl00_ContentPlaceHolder1_ASPxSplitter1_CallbackPaneldgd_dgd_DXPagerBottom']/a[#class='dxp-lead dxp-button dxp-bi']/img[#class='dxWeb_pPrev_Aqua']"))
nextpage.click()