I am new in Python and I am trying to Selenium, however, I have a problem with my code, I can not click the next button on pagination. This is my code:
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Chrome(executable_path='chromedriver.exe')
data = []
page_url = "https://pdb.irb.hr/search?q=&qId=&type=title&limit=2&page=1"
driver.get(page_url)
time.sleep(2)
links = driver.find_elements_by_xpath(
'//*[#id="searchResultList"]/li[2]/p[1]/a')
for x in range(20):
for i in range(len(links)):
driver.find_elements_by_xpath(
'//*[#id="searchResultList"]/li[2]/p[1]/a')[i].click()
time.sleep(2)
title = driver.find_element_by_xpath(
"//html/body/div[2]/div/h2").text
print(title)
data.append((title))
driver.implicitly_wait(5)
driver.execute_script("window.history.go(-1)")
driver.implicitly_wait(5)
driver.find_element_by_css_selector('a[rel="next"]').click()
driver.close()
Can somebody tell me what am doing wrong, how to target that next button?
Thank you
The only problem is to scroll the page till the Pagenumbers get visible.
below code is tested working perfectly fine, was able to navigate next 20 pages.
import time
from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Chrome(executable_path='c:\user1\chromedriver.exe')
data = []
page_url = "https://pdb.irb.hr/search?q=&qId=&type=title&limit=2&page=1"
driver.get(page_url)
driver.maximize_window()
time.sleep(2)
links = driver.find_elements_by_xpath('//*[#id="searchResultList"]/li[2]/p[1]/a')
print(len(links))
for x in range(20):
for i in range(len(links)):
driver.find_elements_by_xpath('//*[#id="searchResultList"]/li[2]/p[1]/a')[i].click()
time.sleep(2)
title = driver.find_element_by_xpath("//html/body/div[2]/div/h2").text
print(title)
data.append((title))
driver.implicitly_wait(5)
driver.execute_script("window.history.go(-1)")
driver.implicitly_wait(5)
below code added to make paginatio visible for selection
button = driver.find_element_by_css_selector('a[rel="next"]')
target = driver.execute_script("arguments[0].scrollIntoView({block: 'center'});", button)
next_btn = driver.find_element_by_css_selector('a[rel="next"]')
next_btn.click()
driver.close()
The next page button is below some other element there so to click on in you have to scroll to it first.
Try this:
from selenium.webdriver.common.action_chains import ActionChains
next_btn = driver.find_element_by_css_selector('a[rel="next"]')
actions = ActionChains(driver)
actions.move_to_element(next_btn).perform()
next_btn.click()
Related
I just need to click the load more button once to reveal a bunch more information so that I can scrape more HTML than what is loaded.
The following "should" go to github.com/topics and find the one and only button element and click it one time.
from selenium import webdriver
from selenium.webdriver.common.by import By
import time
driver = webdriver.Edge()
driver.get("https://github.com/topics")
time.sleep(5)
btn = driver.find_element(By.TAG_NAME, "button")
btn.click()
time.sleep(3)
driver.quit()
I'm told Message: element not interactable so I'm obviously doing something wrong but I'm not sure what.
use
btn = driver.findElementsByXPath("//button[contains(text(),'Load more')]");
You are not finding the right element. This is the reason why it is not "interactable"
There are several issues with your code:
The "Load more" button is initially out of the view, so you have to scroll the page in order to click it.
Your locator is bad.
You need to wait for elements to appear on the page before accessing them. WebDriverWait expected_conditions explicit waits should be used for that, not hardcoded sleeps.
The following code works, it scrolls the page and clicks "Load more" 1 time.
import time
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(options=options, service=webdriver_service)
wait = WebDriverWait(driver, 20)
url = "https://github.com/topics"
driver.get(url)
load_more = wait.until(EC.presence_of_element_located((By.XPATH, "//button[contains(.,'Load more')]")))
load_more.location_once_scrolled_into_view
time.sleep(1)
load_more.click()
UPD
You can simply modify the above code to make it clicking Load more button while it presented.
I implemented this with infinite while loop making a break if Load more button not found. This code works.
import time
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(options=options, service=webdriver_service)
wait = WebDriverWait(driver, 5)
url = "https://github.com/topics"
driver.get(url)
while True:
try:
load_more = wait.until(EC.presence_of_element_located((By.XPATH, "//button[contains(.,'Load more')]")))
load_more.location_once_scrolled_into_view
time.sleep(1)
load_more.click()
except:
break
How to navigate through each page without using driver.current_url? In my full code, I get a bunch of errors once I navigate through the page for a loop. Without it, it runs fine but can only go through one page. I want to navigate through as many pages. Any help appreciated, thanks.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.service import Service
driver_service = Service(executable_path="C:\Program Files (x86)\chromedriver.exe")
driver = webdriver.Chrome(service=driver_service)
driver.maximize_window() # load web driver
wait = WebDriverWait(driver, 5)
url_test = driver.get('https://www.seek.com.au/data-jobs-in-information-communication-technology/in-All-Perth-WA')
url_template = driver.current_url
template = url_template+ '?page={}'
for page in range(2,5):
link_job = [x.get_attribute('href') for x in driver.find_elements(By.XPATH, "//a[#data-automation='jobTitle']")]
for job in link_job:
driver.get(job)
try:
quick_apply = WebDriverWait(driver, 3).until(EC.element_to_be_clickable((By.XPATH, "(//a[#data-automation='job-detail-apply' and #target='_self'])")))
quick_apply.click()
#sleep(3)
except:
print("No records found " + job)
pass
sleep(3)
driver.get(template.format(page))
If I understand you correctly you want to determine dynamically how many pages there are and loop over each of them.
I have managed to achieve this by using a while loop and look on each page if the "Next" button at the bottom is visible. If not, the last page was reached and you can exit the loop.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.service import Service
from time import sleep
driver_service = Service(executable_path="C:\\Users\\Stefan\\bin\\chromedriver.exe")
driver = webdriver.Chrome(service=driver_service)
driver.maximize_window() # load web driver
wait = WebDriverWait(driver, 5)
url_test = driver.get('https://www.seek.com.au/data-jobs-in-information-communication-technology/in-All-Perth-WA')
url_template = driver.current_url
template = url_template+ '?page={}'
page = 1
while True:
# check if "Next" button is visible
# -> if not, the last page was reached
try:
driver.find_element(By.XPATH, "//a[#title='Next']")
except:
# last page reached
break
link_job = [x.get_attribute('href') for x in driver.find_elements(By.XPATH, "//a[#data-automation='jobTitle']")]
for job in link_job:
driver.get(job)
try:
quick_apply = WebDriverWait(driver, 3).until(EC.element_to_be_clickable((By.XPATH, "(//a[#data-automation='job-detail-apply' and #target='_self'])")))
quick_apply.click()
#sleep(3)
except:
print("No records found " + job)
pass
sleep(3)
page += 1
driver.get(template.format(page))
driver.close()
Seems your problem is with StaleElementException when you getting back from job page to jobs search results page.
The simplest approach to overcome this problem is to keep the jobs search results page url.
Actually I changed your code only with this point and it works.
I also changed driver.find_elements(By.XPATH, "//a[#data-automation='jobTitle']") with wait.until(EC.visibility_of_all_elements_located((By.XPATH, "//a[#data-automation='jobTitle']"))) for better performance.
The code below works, but the web site itself responds badly.
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(service=webdriver_service, options=options)
wait = WebDriverWait(driver, 10)
url = 'https://www.seek.com.au/data-jobs-in-information-communication-technology/in-All-Perth-WA?page={p}'
for p in range(1,20):
driver.get(url)
link_job = [x.get_attribute('href') for x in wait.until(EC.visibility_of_all_elements_located((By.XPATH, "//a[#data-automation='jobTitle']")))]
for job in link_job:
driver.get(job)
try:
wait.until(EC.element_to_be_clickable((By.XPATH, "(//a[#data-automation='job-detail-apply' and #target='_self'])"))).click()
print("applied")
except:
print("No records found " + job)
pass
driver.get(url)
If you visit this site,
https://www.premierleague.com/results
You will be able to see several match results. If you click on each match, you will be directed to another website.
My question is how can I get the href (link) of each match.
links = driver.find_elements(By.XPATH, '//*[#id="mainContent"]/div[3]/div[1]')
for link in links:
x = link.get_attribute("href")
List.append(x)
This is what I have so far and it is not working.
I see elements like
<div data-href="//www.premierleague.com/match/66686" ...>
and you could search
//div[#data-href]
and later use get_attribute("data-href")
Full working code
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
#from webdriver_manager.chrome import ChromeDriverManager
from webdriver_manager.firefox import GeckoDriverManager
#import time
url = 'https://www.premierleague.com/results'
#driver = webdriver.Chrome(executable_path=ChromeDriverManager().install())
driver = webdriver.Firefox(executable_path=GeckoDriverManager().install())
driver.get(url)
wait = WebDriverWait(driver, 10)
#time.sleep(5)
# close popup window with "Accept All Cookies"
button = wait.until(EC.visibility_of_element_located((By.XPATH, '//button[text()="Accept All Cookies"]')))
button.click()
all_items = driver.find_elements(By.XPATH, '//div[#data-href]')
print('len(all_items):', len(all_items))
for item in all_items:
print(item.get_attribute('data-href'))
Result:
len(all_items): 40
//www.premierleague.com/match/66686
//www.premierleague.com/match/66682
//www.premierleague.com/match/66687
//www.premierleague.com/match/66689
//www.premierleague.com/match/66691
//www.premierleague.com/match/66684
//www.premierleague.com/match/66705
//www.premierleague.com/match/66677
//www.premierleague.com/match/66674
//www.premierleague.com/match/66675
//www.premierleague.com/match/66676
//www.premierleague.com/match/66679
//www.premierleague.com/match/66672
//www.premierleague.com/match/66678
//www.premierleague.com/match/66680
//www.premierleague.com/match/66681
//www.premierleague.com/match/66673
//www.premierleague.com/match/66633
//www.premierleague.com/match/66584
//www.premierleague.com/match/66513
//www.premierleague.com/match/66637
//www.premierleague.com/match/66636
//www.premierleague.com/match/66635
//www.premierleague.com/match/66666
//www.premierleague.com/match/66670
//www.premierleague.com/match/66668
//www.premierleague.com/match/66665
//www.premierleague.com/match/66667
//www.premierleague.com/match/66669
//www.premierleague.com/match/66654
//www.premierleague.com/match/66656
//www.premierleague.com/match/66659
//www.premierleague.com/match/66657
//www.premierleague.com/match/66655
//www.premierleague.com/match/66652
//www.premierleague.com/match/66660
//www.premierleague.com/match/66661
//www.premierleague.com/match/66653
//www.premierleague.com/match/66658
//www.premierleague.com/match/66524
im trying to print all links but i have an error ( ut is not clickable at point (781,748) because another element obscures it
the code updated:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
import time
#driver = webdriver.Chrome(executable_path='chromedriver.exe')
driver = webdriver.Firefox(executable_path='geckodriver')
wait = WebDriverWait(driver, 20)
actions = ActionChains(driver)
driver.get("https://www.architectes-pour-tous.fr/")
driver.find_element_by_xpath("//button[contains(#class,'decline-button')]").click();
driver.find_element_by_xpath(".//a[#id='pager']").click();
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "div.image-projet img")))
time.sleep(1)
for img in driver.find_elements_by_css_selector('div.image-projet img'):
print(a.get_attribute('href'))
driver.find_elements_by_css_selector('button.agree-button').click()
pager = driver.find_element_by_xpath('//*[#id="pager"]')
actions.move_to_element(pager).build().perform()
time.sleep(0.5)
pager.click()
You have to accept/decline the cookies before accessing any element on the page.
driver.find_element_by_xpath("//button[contains(#class,'decline-button')]").click();
driver.find_element_by_xpath(".//a[#id='pager']").click();
The element you trying to access is initially out of the visible screen so you have to scroll to it before clicking it.
Also possibly you will have to close the accept cookies pop-up prior to clicking this element.
Also I'm quite sure you are getting no links with
for a in driver.find_elements_by_xpath('.//a'):
print(a.get_attribute('href'))
since you trying to do that before the page is loaded.
Also, if you are trying to get the search results links you have to use another locator.
So would suggest to change your code as following:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
import time
#driver = webdriver.Chrome(executable_path='chromedriver.exe')
driver = webdriver.Firefox(executable_path='geckodriver')
wait = WebDriverWait(driver, 20)
actions = ActionChains(driver)
driver.get("https://www.architectes-pour-tous.fr/")
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "div.image-projet img")))
time.sleep(1)
for img in driver.find_elements_by_css_selector('div.image-projet img'):
print(a.get_attribute('href'))
driver.find_element_by_css_selector('button.agree-button').click()
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
pager = driver.find_element_by_xpath('//*[#id="pager"]')
actions.move_to_element(pager).perform()
time.sleep(0.5)
pager.click()
I am trying to click spam button with Selenium on Python. But when I go to three dotes on someone profile i can click report #thisperson then I can't click "It’s suspicious or spam" button. I'm going crazy because I tried every way for pulling button with selenium. For example, find_element_by_id, path, css selector, class name and They don't work. Please help me,show me a way, enlighten me.
My Python code for clicking button:
spam_button = browser.find_element_by_xpath('//form[#id="report_webview_form"]/button[2]/span')
here is html tags for this button :
I tried a complicated path for clicking button :
spam_button = browser.find_element_by_xpath('//*[#id="react-root"]/div/div/div[2]/main/div/div/div[2]/div/iframe/#document/html/body/div/div/form/button[2]/span')
I don't know how I write "document" tag. So I get the error.
I tried find_element_by_id:
spam_button = browser.find_element_by_id('spam-btn')
browser.implicitly_wait(5)
ActionChains(browser).move_to_element(spam_button).click(spam_button).perform()
And I got the error "Unable element"
I am lost. How can I click the button with selenium ?
My whole code:
import time
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
browser = webdriver.Chrome(ChromeDriverManager().install())
url ="https://twitter.com/ANYACCOUNT"
browser.get(url)
time.sleep(1)
browser.find_element_by_xpath("//a[#href='/login']").click()
time.sleep(1)
username = browser.find_element_by_xpath('//*[#id="react-root"]/div/div/div[2]/main/div/div/div[2]/form/div/div[1]/label/div/div[2]/div/input')
time.sleep(1)
username.send_keys("MYNICKNAME")
time.sleep(1)
password = browser.find_element_by_xpath('//*[#id="react-root"]/div/div/div[2]/main/div/div/div[2]/form/div/div[2]/label/div/div[2]/div/input')
time.sleep(1)
password.send_keys("MYPASSWORD")
time.sleep(1)
browser.find_element_by_xpath('//*[#id="react-root"]/div/div/div[2]/main/div/div/div[2]/form/div/div[3]/div/div/span/span').click()
time.sleep(1)
button = browser.find_element_by_xpath('//*[#id="react-root"]/div/div/div[2]/main/div/div/div/div/div/div[2]/div/div/div/div/div/div/div/div/span')
browser.implicitly_wait(5)
ActionChains(browser).move_to_element(button).click(button).perform()
time.sleep(1)
sikayet = browser.find_element_by_xpath('//*[#id="react-root"]/div/div/div/div[2]/div/div/div/div[2]/div[3]/div/div/div/div[5]/div[2]/div/span')
browser.implicitly_wait(5)
ActionChains(browser).move_to_element(sikayet).click(sikayet).perform()
time.sleep(1)
#spam_button = browser.find_element_by_xpath('//*[#id="react-root"]/div/div/div[2]/main/div/div/div[2]/div/iframe/#document/html/body/div/div/form/button[2]/span')
#browser.find_element_by_xpath('//button[#id="spam-btn"]')
#browser.implicitly_wait(5)
#ActionChains(browser).move_to_element(spam_button).click(spam_button).perform()
wait = WebDriverWait(browser,10 )
spam_button = wait.until(EC.element_to_be_clickable((By.ID, "spam-btn")))
time.sleep(1)
spam_button.click()
I tried other ways as mentioned in below comment and I got this error everytime .
Try using explicit wait instead of implicit:
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
wait = WebDriverWait(driver, 20)
wait.until(EC.element_to_be_clickable((By.ID, "spam-btn")))
spam_button = browser.find_element_by_id('spam-btn')
spam_button.click()
OR,
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
wait = WebDriverWait(driver, 20)
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "#spam-btn>div")))
spam_button = browser.find_element_by_css_selector('#spam-btn>div')
spam_button.click()
Update:
You need to switch to iframe.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
driver = webdriver.Chrome(executable_path='/snap/bin/chromium.chromedriver')
driver.get('https://twitter.com/apiontkovsky/status/1385614545947820037')
wait = WebDriverWait(driver, 15)
# Authorise here
# ...
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, ".css-1dbjc4n.r-18u37iz.r-15zivkp .css-1dbjc4n.r-xoduu5 svg")))
driver.find_element_by_css_selector(".css-1dbjc4n.r-18u37iz.r-15zivkp .css-1dbjc4n.r-xoduu5 svg").click()
wait.until(EC.element_to_be_clickable((By.XPATH, "//span[contains(text(),'Report')]")))
driver.find_element_by_xpath("//span[contains(text(),'Report')]").click()
wait.until(EC.frame_to_be_available_and_switch_to_it((By.CSS_SELECTOR, ".r-1yadl64.r-16y2uox")))
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "#spam-btn>div")))
spam_button = driver.find_element_by_css_selector('#spam-btn>div')
spam_button.click()