How do I paginate to the next page in selenium python?
Code trials:
import os,json,requests, time
from pprint import pprint
from selenium import webdriver
import csv
driver = webdriver.Chrome()
driver.get('https://www.ebay.co.uk/b/Cordless-Drills/184655/bn_9863722?rt=nc&_dmd=1')
time.sleep(0.5)
links = driver.find_elements_by_xpath('//div[#class="s-item__wrapper clearfix"]//a[#class="s-item__link"]')
for link in links:
print(link.get_attribute('href'))
To paginate to the next page you need to induce WebDriverWait for the element_to_be_clickable() and you can use either of the following Locator Strategies:
Using CSS_SELECTOR:
driver.get('https://www.ebay.co.uk/b/Cordless-Drills/184655/bn_9863722?rt=nc&_dmd=1')
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button#gdpr-banner-accept[aria-label='Accept privacy terms and settings']"))).click()
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "li.ebayui-pagination__li.ebayui-pagination__li--selected +li > a"))).click()
Using XPATH:
driver.get('https://www.ebay.co.uk/b/Cordless-Drills/184655/bn_9863722?rt=nc&_dmd=1')
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//button[#id='gdpr-banner-accept']"))).click()
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//li[#class='ebayui-pagination__li ebayui-pagination__li--selected']//following::li[1]/a"))).click()
Note: You have to add the following imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
Browser Snapshot:
Related
I'm trying to click the next button (>) and then repeat until the last page is reached. I've tried searching for solutions from similar questions but I couldn't figure out what's wrong/make it work.
Here is my code, thank you!
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait as Wait
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import TimeoutException
driver = webdriver.Chrome("C:/Users/krish/Desktop/chromedriver_win32/chromedriver.exe")
driver.get('https://www.cnbcindonesia.com/tag/pasar-modal')
wait = WebDriverWait(driver,30)
while True:
try:
element = wait.until(EC.element_to_be_clickable((By.XPATH, "//a[#class = 'icon icon-angle-right']/i")))
if (element != 0):
element.click()
except TimeoutException as ex:
break
Figured it out, finally works by using execute_script() method
while True:
try:
time.sleep(2)
driver.execute_script("arguments[0].click();", WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, ".icon.icon-angle-right"))))
except NoSuchElementException:
break
Thank you all for your answers
Your xpath looks incorrect, please try the below
Xpath
//i[#class='icon icon-angle-right']
CSS Selector
.icon.icon-angle-right
There are multiple approaches to address this issue and a couple of them are as follows:
As you intent to invoke click() you need to induce WebDriverWait in conjunction with the WebDriverWaitWebDriverWait for the element_to_be_clickable() and you can use either of the following Locator Strategies:
Using CSS_SELECTOR:
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, ".icon.icon-angle-right"))).click()
Using XPATH:
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//i[#class='icon icon-angle-right']"))).click()
Incase the error ...another element obscures it... still persists first you need to induce WebDriverWait inconjunction with the expected_conditions for the invisibility_of_element() of the blocking element as follows:
Using CSS_SELECTOR:
WebDriverWait(driver, 20).until(EC.invisibility_of_element((By.CSS_SELECTOR, ".icon.icon-angle-right")))
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, ".icon.icon-angle-right"))).click()
Using XPATH:
WebDriverWait(driver, 20).until(EC.invisibility_of_element((By.XPATH, "//i[#class='icon icon-angle-right']")))
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//i[#class='icon icon-angle-right']"))).click()
If the issue still persists you can use the execute_script() method as follows:
Using CSS_SELECTOR:
WebDriverWait(driver, 20).until(EC.invisibility_of_element((By.CSS_SELECTOR, ".icon.icon-angle-right")))
driver.execute_script("arguments[0].click();", WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, ".icon.icon-angle-right"))))
Using XPATH:
WebDriverWait(driver, 20).until(EC.invisibility_of_element((By.XPATH, "//i[#class='icon icon-angle-right']")))
driver.execute_script("arguments[0].click();", WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//i[#class='icon icon-angle-right']"))))
Note: : You have to import below
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
You can do it easy with playwright
from playwright.sync_api import sync_playwright
with sync_playwright() as p:
browser = p.webkit.launch(headless=False)
baseurl = "https://www.cnbcindonesia.com/tag/pasar-modal"
page = browser.new_page()
page.goto(baseurl)
next_page = page.wait_for_selector("//i[#class='icon icon-angle-right']")
next_page.click()
browser.close()
Also if you want to paginate, you can do it with changing just URL param.
Go to page 87: https://www.cnbcindonesia.com/tag/pasar-modal/87?kanal=&tipe=
I'm trying to click the play button on this website
Here is my code below
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
import time
browser = webdriver.Chrome()
browser.get("https://audiomack.com/iamrealtraffic")
time.sleep(10)
a = driver.find_element_by_xpath('//path[#d="M70.79 103c-.988 0-1.79-.802-1.79-1.79V69.866c0-.99.802-1.79 1.79-1.79l29.104 16.118s1.344 1.343 0 2.686C98.55 88.225 70.79 103 70.79 103"]')
webdriver.ActionChains(browser).move_to_element(a).click(a).perform()
Are you sure you have the right xpath?
I would try this xpath:
/html/body/div[2]/div[3]/div/div[2]/div[1]/div/div/div/div/div/div[2]/div/div/span[1]/button
Also I would use implicitlywait instead of time.sleep.
To click on the play icon you need to induce WebDriverWait for the element_to_be_clickable() and you can use either of the following locator strategies:
Using CSS_SELECTOR:
driver.get("https://audiomack.com/black-sherif/song/kweku-the-traveler")
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div.row button[data-action='play']"))).click()
Using XPATH:
driver.get("https://audiomack.com/black-sherif/song/kweku-the-traveler")
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//div[#class='row']//button[#data-action='play']"))).click()
Note: You have to add the following imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
I am using Selenium with Python to webscrape a page which includes JavaScript.
The racecourse result tabs towards the top of the page eg "Ludlow","Dundalk" are manually clickable but do not have any obvious hyperlinks attached to them.
...
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Chrome(executable_path='C:/A38/chromedriver_win32/chromedriver.exe')
driver.implicitly_wait(30)
driver.maximize_window()
# Navigate to the application home page
driver.get("https://www.sportinglife.com/racing/results/2020-11-23")
...
This works so far. I have used BeautifulSoup to find the label names of the NewGenericTabs eg "Ludlow","Dundalk" etc. However, the following code, to attempt to automate clicking a tab, times out each time.
WebDriverWait(driver, 60).until(EC.element_to_be_clickable((By.LINK_TEXT, "Ludlow"))).click()
Would welcome any help.
The WebElement aren't <a> tags but <span> tags so By.LINK_TEXT wouldn't work.
To click on the desired elements you can use either of the following xpath based Locator Strategies:
Ludlow:
driver.get("https://www.sportinglife.com/racing/results/2020-11-23")
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//button[#mode='primary']"))).click()
driver.find_element_by_xpath("//span[#data-test-id='generic-tab' and text()='Ludlow']").click()
Dundalk:
driver.get("https://www.sportinglife.com/racing/results/2020-11-23")
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//button[#mode='primary']"))).click()
driver.find_element_by_xpath("//span[#data-test-id='generic-tab' and text()='Dundalk']").click()
Ideally, to click on the elements you need to induce WebDriverWait for the element_to_be_clickable() and you can use the following xpath based Locator Strategies:
Ludlow:
driver.get("https://www.sportinglife.com/racing/results/2020-11-23")
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//button[#mode='primary']"))).click()
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//span[#data-test-id='generic-tab' and text()='Ludlow']"))).click()
Dundalk:
driver.get("https://www.sportinglife.com/racing/results/2020-11-23")
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//button[#mode='primary']"))).click()
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//span[#data-test-id='generic-tab' and text()='Dundalk']"))).click()
Note: You have to add the following imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
I am trying to crawl the reviews on this website: https://www.bol.com/nl/p/Matras-140x200-7-zones-koudschuim-premium-plus-tijk-15-cm-medium/9200000118425897/.
However, I have to click a button ( Toon meer) to show all the reviews.
<div class="load-more load-more--divider load-more--reviews js-review-load-more-container">
<a data-href="/nl/rnwy/productPage/reviews?productId=9200000118425896&offset=5&limit=10&loadMore=true" class="review-load-more__button js-review-load-more-button" data-test="review-load-more"><div class="css-loader css-loader--reviews"></div>
Toon meer</a>
</div>
I use the below code :
import requests
import pandas as pd
from selenium import webdriver
from bs4 import BeautifulSoup
from datetime import datetime
start_time = datetime.now()
data = []
link = "https://www.bol.com/nl/p/Matras-140x200-7-zones-koudschuim-premium-plus-tijk-15-cm-medium/9200000118425897/"
op = webdriver.ChromeOptions()
op.add_argument('--ignore-certificate-errors')
op.add_argument('--incognito')
op.add_argument('--headless')
driver = webdriver.Chrome(executable_path='D:/Desktop/work/real/chromedriver.exe',options=op)
driver.get(link)
driver.find_element_by_css_selector('div.review-load-more__button js-review-load-more-button').click()
However, it throws an error:
No such element: Unable to locate element: {"method":"css selector","selector":"div.review-load-more__button js-review-load-more-button"} .
Is there any solution?
Css selectors cannot select an element by containing text.
Try using xpath. The last line of your script should look something like:
wait = WebDriverWait(driver, 10)
wait.until(expected_conditions.element_to_be_clickable((By.XPATH, "//a[contains(., 'Toon meer')]")).click()
To click on Toon meer you need to induce WebDriverWait for the element_to_be_clickable() and you can use either of the following Locator Strategies:
Using CSS_SELECTOR:
driver.get('https://www.bol.com/nl/p/Matras-140x200-7-zones-koudschuim-premium-plus-tijk-15-cm-medium/9200000118425896/')
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button[data-test='consent-modal-confirm-btn']>span"))).click()
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "a.review-load-more__button.js-review-load-more-button"))).click()
Using XPATH:
driver.get('https://www.bol.com/nl/p/Matras-140x200-7-zones-koudschuim-premium-plus-tijk-15-cm-medium/9200000118425896/')
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//button[#data-test='consent-modal-confirm-btn']/span"))).click()
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//a[#class='review-load-more__button js-review-load-more-button' and contains(., 'Toon meer')]"))).click()
Note: You have to add the following imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
When you get the page a popup comes with an accept button click it and then proceed with clicking your element.
driver.get('https://www.bol.com/nl/p/Matras-140x200-7-zones-koudschuim-premium-plus-tijk-15-cm-medium/9200000118425896/')
wait=WebDriverWait(driver, 10)
wait.until(EC.element_to_be_clickable((By.XPATH, "//button[#class='js-confirm-button']"))).click()
wait.until(EC.element_to_be_clickable((By.XPATH, "//a[#data-test='review-load-more']"))).click()
Import
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
How do I convert the above to selenium xpath?
I tried:
"//ul[#id = 'productsCatalog AND #class = 'b-catalogList_wrapper clearfix']"
but it gives an error
As per the HTML you have shared the element is a dynamic element so you have to induce WebDriverWait for the element to be visible and you can use either of the following solutions:
CSS_SELECTOR:
element = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "ul.b-catalogList_wrapper.clearfix#productsCatalog")))
XPATH:
element = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//ul[#class = 'b-catalogList_wrapper clearfix' and #id ='productsCatalog']")))
Note: You have to add the following imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC