I am using the following code to click on the next page, by clicking the next element.
However this code is not working. Any thoughts on what I might be doing wrong.
Final goal:
Use BeautifulSoup on each of the pages.
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup as soup
from urllib.request import urlopen as ureq
import random
import time
from bs4 import BeautifulSoup
from selenium.webdriver.common.by import By
chrome_options = webdriver.ChromeOptions()
prefs = {"profile.default_content_setting_values.notifications": 2}
chrome_options.add_experimental_option("prefs", prefs)
# A randomizer for the delay
seconds = 1 + (random.random() * 2)
# create a new Chrome session
driver = webdriver.Chrome(chrome_options=chrome_options)
driver.implicitly_wait(30)
# driver.maximize_window()
# navigate to the application home page
driver.get("https://www.fda.gov/inspections-compliance-enforcement-and-criminal-investigations/compliance-actions-and-activities/warning-letters")
time.sleep(seconds)
time.sleep(seconds)
next_page = driver.find_element(By.CLASS_NAME, "next")
#print (next_page.get_attribute('innerHTML'), type(next_page))
next_page.find_element(By.XPATH("//a[#href='#']")).click
# next_page.find_element(By.LINK_TEXT("Next")).click()
This code does not click on the next page.
Select your element more specific and wait .until(EC.element_to_be_clickable()) - This will give you the next page.
Example
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()))
url = 'https://www.fda.gov/inspections-compliance-enforcement-and-criminal-investigations/compliance-actions-and-activities/warning-letters'
driver.get(url)
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, '#datatable_next a'))).click()
Related
I am not getting price they give me empty output this is page link https://www.amazon.com/dp/B00M0DWQYI?th=1
from selenium import webdriver
import time
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support.select import Select
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
url='https://www.amazon.com/dp/B00M0DWQYI?th=1'
PATH="C:\Program Files (x86)\chromedriver.exe"
driver =webdriver.Chrome(PATH)
driver.get(url)
item=dict()
try:
item['price'] = driver.find_element(By.XPATH, "//div[#id='corePrice_feature_div'] //span[#class='a-offscreen']").text
except:
item['price']=''
print(item)
You may want to wait for that element to properly load, prior to locating it:
[...]
wait = WebDriverWait(driver, 10)
item['price'] = wait.until(EC.element_to_be_clickable((By.XPATH, "//div[#id='corePrice_feature_div']//span[#class='a-offscreen']"))).text
Selenium documentation can be found at https://www.selenium.dev/documentation/
EDIT: Here is a complete example of how you can get that information:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
import time as t
chrome_options = Options()
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument('disable-notifications')
chrome_options.add_argument("window-size=1920,1080")
webdriver_service = Service("chromedriver/chromedriver") ## path to where you saved chromedriver binary
driver = webdriver.Chrome(service=webdriver_service, options=chrome_options)
wait = WebDriverWait(driver, 5)
items = dict()
driver.get('https://www.amazon.com/dp/B00M0DWQYI?th=1')
t.sleep(1)
driver.refresh()
items['price'] = wait.until(EC.element_to_be_clickable((By.XPATH, '//div[#id="corePrice_feature_div"]//span[#class="a-price aok-align-center"]'))).text.replace('\n', '.')
print(items)
Result in terminal:
{'price': '$32.98'}
You need to wait for element visibility and then to extract it's text.
The following Selenium code works:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(service=webdriver_service, options=options)
url = 'https://www.amazon.com/dp/B00M0DWQYI'
driver.get(url)
wait = WebDriverWait(driver, 10)
print(wait.until(EC.visibility_of_element_located((By.XPATH, "//div[#id='corePrice_feature_div']"))).text)
The output is
$32
98
You can use bs4 and it will work fine
from bs4 import BeautifulSoup
soup = BeautifulSoup(driver.page_source, 'lxml')
try:
item['price'] = soup.find('input', id="attach-base-product-price").get('value')
except:
item['price'] = ''
finally:
driver.close()
driver.quit()
print(item)
https://www.espncricinfo.com/player/aamer-jamal-793441
This is the URL and here i am trying to access Full Name "Aamer Jamal". with the help of selenium web driver. But I dont know why it gives
NoSuchElementException
`the code is written below:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium_firefox import Firefox
import time
import pandas as pd
driver = webdriver.Firefox()
#Reach to the Landing page
driver.get('https://www.espncricinfo.com/player/aamer-jamal-793441')
driver.maximize_window()
time.sleep(25)
not_now = driver.find_element(By.ID, 'wzrk-cancel')
not_now.click()
fullname = driver.find_element(By.XPATH, '/html/body/div[1]/section/section/div[4]/div[2]/div/div[1]/div/div/div[1]/div[1]/span/h5')
print(fullname.text)`
Error :
NoSuchElementException: Message: Unable to locate element: /html/body/div[1]/section/section/div[4]/div[2]/div/div[1]/div/div/div[1]/div[1]/span
You have to use WebDriverWait expected_conditions explicit waits, not a long hardcoded pauses. You also have to learn how to create correct locators. Long absolute XPaths and CSS Selectors are extremely breakable. The following code works:
from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(options=options, service=webdriver_service)
wait = WebDriverWait(driver, 60)
actions = ActionChains(driver)
url = "https://www.espncricinfo.com/player/aamer-jamal-793441"
driver.get(url)
wait.until(EC.element_to_be_clickable((By.ID, 'wzrk-cancel')))
fullname = wait.until(EC.visibility_of_element_located((By.CLASS_NAME, 'ds-text-title-l'))).text
print(fullname)
The output is:
Aamer Jamal
I was working on a web scraping project with Selenium and was trying to scrape news from the site https://www.businesstimes.com.sg/government-economy.
But whenever I open the site with the selenium automated chrome window, one ad comes up in a popup which I want to close.
import selenium
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as LM
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.support.ui import Select
import pandas as pd
import time
options = webdriver.ChromeOptions()
"""options.add_argument("enable-automation")
options.add_argument("--headless")"""
lists = ['disable-popup-blocking']
caps = DesiredCapabilities().CHROME
caps["pageLoadStrategy"] = "normal"
options.add_argument("--window-size=1920,1080")
options.add_argument("--disable-extensions")
options.add_argument("--disable-notifications")
options.add_argument("--disable-Advertisement")
options.add_argument("--disable-popup-blocking")
driver = webdriver.Chrome(executable_path= r"E:\chromedriver\chromedriver.exe", options=options, desired_capabilities=caps) #paste your own choromedriver path
driver.get('https://www.businesstimes.com.sg/government-economy')
I tried two methods, one was by the xpath method and one by the CSS selector method, but both failed.
#1st method
driver.find_element(By.CSS_SELECTOR, 'button[data-id="pclose-btn"]').click()
#2nd method
driver.find_element_by_xpath("//div[#class='bz-el bz-pclose-btn knd-BUTTON']").click()
Please help me with this. Thank you!
Element you trying to access is inside nested iframe.
This should work:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(service=webdriver_service, options=options)
url = 'https://www.businesstimes.com.sg/government-economy'
driver.get(url)
wait = WebDriverWait(driver, 20)
wait.until(EC.frame_to_be_available_and_switch_to_it((By.CSS_SELECTOR, "iframe[id*='prestitial']")))
wait.until(EC.frame_to_be_available_and_switch_to_it((By.CSS_SELECTOR, "iframe")))
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button[data-id='pclose-btn']"))).click()
I'm trying to get a screen shot from of a website from an url
however how can I accept the cookies before the screen shot ?
If you have any idea or clue I will appreciate.
Here is a following picture example of cookies :
Here is my code:
import time
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
start = time.time()
browser = webdriver.Chrome(ChromeDriverManager().install())
browser.get("https://candidat.pole-emploi.fr/offres/recherche/detail/136LNXS/")
browser.save_screenshot('screenshot1.png')
browser.quit()
end = time.time()
print("tempo =","\t",end - start)
You have to click on accept-button to accept cookie
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
import time
# selenium 4
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
#from selenium.webdriver.chrome.options import Options
options = webdriver.ChromeOptions()
options.add_argument("start-maximized")
options.add_argument("--no-sandbox")
options.add_argument("--disable-gpu")
options.add_argument("--disable-extensions")
#chrome to stay open
options.add_experimental_option("detach", True)
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()),options=options)
driver.get('https://candidat.pole-emploi.fr/offres/recherche/detail/136LNXS')
WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.XPATH, '//*[#id="footer_tc_privacy_button_2"]'))).click()
I'm having a problem clicking the Radio button for the Registered Projects on this site. It is not clicking with my code in selenium webdriver.
import urllib.request
from bs4 import BeautifulSoup
import os
from selenium import webdriver
from selenium.webdriver.support.select import Select
from selenium.webdriver.common.keys import Keys
url = 'https://maharerait.mahaonline.gov.in'
chrome_path = r'C:/Users/User/AppData/Local/Programs/Python/Python36/Scripts/chromedriver.exe'
driver = webdriver.Chrome(executable_path=chrome_path)
driver.implicitly_wait(10)
driver.get(url)
soup=BeautifulSoup(driver.page_source, 'lxml')
link =driver.find_element_by_link_text("Search Project Details")
link.click()
driver.find_element_by_id("Promoter").click()
Use WebDriverWait and java Scripts Executor to click on the Registered Project radio button.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
url = 'https://maharerait.mahaonline.gov.in'
chrome_path = r'C:/Users/User/AppData/Local/Programs/Python/Python36/Scripts/chromedriver.exe'
driver = webdriver.Chrome(executable_path=chrome_path)
driver.get(url)
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH,"//div[#class='search-pro-details']//a[contains(.,'Search Project Details')]"))).click()
Registered_Project_radio= WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID,"Promoter")))
driver.execute_script("arguments[0].click();",Registered_Project_radio)
Browser snapshot:
WebDriverWait - An explicit wait is a code you define to wait for a certain condition to occur before proceeding further in the code.
import urllib.request
from bs4 import BeautifulSoup
import os
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.select import Select
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
url = 'https://maharerait.mahaonline.gov.in'
chrome_path = r'C:/Users/User/AppData/Local/Programs/Python/Python36/Scripts/chromedriver.exe'
driver = webdriver.Chrome(executable_path=chrome_path)
driver.get(url)
links = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CLASS_NAME,\
"search-pro-details")))
#Click on Search Project Details link
links.find_element_by_link_text("Search Project Details").click()
promoter_radio_button = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID,\
"Promoter")))
#select radio button
promoter_radio_button.send_keys(Keys.SPACE)
Try using this:
driver.get('https://maharerait.mahaonline.gov.in')
link =driver.find_element_by_link_text("Search Project Details")
link.click()
time.sleep(2)
radio_btn = driver.find_element_by_id("Promoter")
radio_btn.click()
time.sleep(5)
driver.close()