I don't know how to crawl the title of the page,below is my code(it's simple),but I have no idea where is wrong, if you have any idea please let me know,thank you.
from selenium import webdriver
url="https://sukebei.nyaa.si/?s=seeders&o=desc&p=1"
driver_path = "C:\\webdriver\\chromedriver.exe"
option = webdriver.ChromeOptions()
driver = webdriver.Chrome(driver_path, options=option)
driver.implicitly_wait(10)
driver.get(url)
print(driver.find_element_by_xpath("/html/head/title").text)
To crawl the title of the page you have to induce WebDriverWait for the visibility_of_element_located() for the <table> with torrent-list and you can use either of the following Locator Strategies:
Using CSS_SELECTOR:
driver.get('https://sukebei.nyaa.si/?s=seeders&o=desc&p=1')
WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.CSS_SELECTOR, "table.torrent-list")))
print(driver.title)
Using XPATH:
driver.get('https://sukebei.nyaa.si/?s=seeders&o=desc&p=1')
WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.XPATH, "//table[contains(#class, 'torrent-list')]")))
print(driver.title)
Console Output:
Browse :: Sukebei
Note : You have to add the following imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium import webdriver
url="https://sukebei.nyaa.si/?s=seeders&o=desc&p=1"
driver_path = "C:\\webdriver\\chromedriver.exe"
option = webdriver.ChromeOptions()
driver = webdriver.Chrome(driver_path, options=option)
driver.implicitly_wait(10)
driver.get(url)
print(driver.title)
Related
I want to find out the "Accept All" button xpath for click accept cookies.
Code trials:
from ast import Pass
import time
from selenium import webdriver
driver = driver = webdriver.Chrome(executable_path=r'C:\Users\Nahid\Desktop\Python_code\Jobsite\chromedriver.exe') # Optional argument, if not specified will search path.
driver.get('http://jobsite.co.uk/')
driver.maximize_window()
time.sleep(1)
#find out XPath in div tag but there has another span tag
cookie = driver.find_element_by_xpath('//div[#class="privacy-prompt-button primary-button ccmgt_accept_button "]/span')
cookie.click()
The desired element:
<div id="ccmgt_explicit_accept" class="privacy-prompt-button primary-button ccmgt_accept_button ">
<span>Accept All</span>
</div>
is a <span> tag having an ancestor <div>.
Solution
To click on the clickable element you need to induce WebDriverWait for the element_to_be_clickable() and you can use either of the following locator strategies:
Using CSS_SELECTOR:
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div.privacy-prompt-button.primary-button.ccmgt_accept_button>span"))).click()
Using XPATH:
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//span[text()='Accept All']"))).click()
Note: You have to add the following imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
Your XPath looks correct but if can be improved.
Also you should use WebDriverWait expected conditions instead of hardcoded sleeps.
As following:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("--start-maximized")
s = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(options=options, service=s)
url = 'http://jobsite.co.uk/'
wait = WebDriverWait(driver, 10)
driver.get(url)
wait.until(EC.element_to_be_clickable((By.ID, "ccmgt_explicit_accept"))).click()
I've tried using the elements linktext, value and xpath. I cant seem to make it click on the button with anything. What am I doing wrong?
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support import expected_conditions as EC
import time
PATH = "C:/Users/yongs/Downloads/chromedriver_win32/chromedriver.exe"
driver = webdriver.Chrome(PATH)
driver.get("https://ttsfree.com/")
textbox = driver.find_element("id", "input_text")
textbox.send_keys("Text to convert")
driver.implicitly_wait(5)
button_xpath = "/html/body/section[2]/div[2]/form/div[2]/div[2]/a"
button = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, button_xpath)))
actions = ActionChains(driver)
actions.click(button)
actions.perform()
EDITED to include download solution as well
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.relative_locator import locate_with
import time as t
chrome_options = Options()
chrome_options.add_argument("--no-sandbox")
# chrome_options.add_argument("--headless")
webdriver_service = Service("chromedriver/chromedriver") ## path to where you saved chromedriver binary
browser = webdriver.Chrome(service=webdriver_service, options=chrome_options)
url = 'https://ttsfree.com/'
browser.get(url)
WebDriverWait(browser, 20).until(EC.element_to_be_clickable((By.XPATH, "//button/span[text()='AGREE']"))).click()
textbox = WebDriverWait(browser, 20).until(EC.element_to_be_clickable((By.ID, "input_text")))
textbox.send_keys("Text to convert")
button = WebDriverWait(browser, 2000).until(EC.element_to_be_clickable((By.XPATH, "//a[text()='Convert Now']")))
button.click()
print('clicked!')
t.sleep(10)
browser.execute_script('document.getElementsByClassName("label_process text-left")[0].scrollIntoView();')
dl_button = WebDriverWait(browser, 2000).until(EC.element_to_be_clickable((By.CLASS_NAME, "fa-download")))
dl_button.click()
t.sleep(10)
browser.quit()
To click on the element Convert Now you need to induce WebDriverWait for the element_to_be_clickable() which automatically scrolls the element within view and you can use the following locator strategy:
Using LINK_TEXT:
driver.execute("get", {'url': 'https://ttsfree.com/'})
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "textarea#input_text"))).send_keys("Hello")
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.LINK_TEXT, "Convert Now"))).click()
Using CSS_SELECTOR:
driver.execute("get", {'url': 'https://ttsfree.com/'})
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "textarea#input_text"))).send_keys("Hello")
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "a.convert-now"))).click()
Using XPATH:
driver.execute("get", {'url': 'https://ttsfree.com/'})
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "textarea#input_text"))).send_keys("Hello")
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//a[text()='Convert Now']"))).click()
Note: You have to add the following imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
Im trying to search for stock on www.finanzen.net using selenium but always get
ElementNotInteractableException: element not interactable
from selenium import webdriver
import time
chrome_options = webdriver.ChromeOptions()
prefs = {"profile.default_content_setting_values.notifications" : 2}
chrome_options.add_experimental_option("prefs", prefs)
chrome_options.add_argument("start-maximized")
driver = webdriver.Chrome(options=chrome_options, executable_path=r'F:\chromedriver.exe')
driver.get('https://www.finanzen.net/')
time.sleep(5)
cookie_banner_button = driver.find_element_by_xpath("//button[#onclick='cookieBannerOverlayClick();']")
cookie_banner_button.click()
search_field = driver.find_element_by_xpath("//input[#class='search-input']")
#search_field.click()
search_field.send_keys('bmw')
search_field.submit()
time.sleep(5)
driver.quit()
HTML:
The xpath that you have used are pointing to two elements and first element on which it is pointing is not interactable due to which you are getting the exception.
Have found the correct xpath for the element, please refer to the code below:
from selenium import webdriver
import time
chrome_options = webdriver.ChromeOptions()
prefs = {"profile.default_content_setting_values.notifications" : 2}
chrome_options.add_experimental_option("prefs", prefs)
chrome_options.add_argument("start-maximized")
driver = webdriver.Chrome(options=chrome_options,
executable_path=r'F:\chromedriver.exe')
driver.get('https://www.finanzen.net/')
time.sleep(5)
cookie_banner_button = driver.find_element_by_xpath("//button[#onclick='cookieBannerOverlayClick();']")
cookie_banner_button.click()
search_field = driver.find_element_by_xpath("//div[#class='shadow']//input[#class='search-input']")
search_field.send_keys('bmw')
search_field.submit()
time.sleep(5)
driver.quit()
To search for stock on www.finanzen.net you need to induce WebDriverWait for the element_to_be_clickable() and you can use either of the following Locator Strategies:
Using CSS_SELECTOR:
driver.get('https://www.finanzen.net/')
submit_button = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "form[name='mmssearch'] div.search-group>input.search-input[name='_search']")))
submit_button.send_keys('bmw')
submit_button.submit()
Using XPATH:
driver.get('https://www.finanzen.net/')
submit_button = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//form[#name='mmssearch']//div[#class='search-group']/input[#class='search-input' and #name='_search']")))
submit_button.send_keys('bmw')
submit_button.submit()
Note : You have to add the following imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
Browser Snapshot:
I am really new to selenium.
Currently, I am trying to use both selenium and beautifulsoup to do some webcrawling. The website that I am webcrawling on is https://bigd.big.ac.cn/dogsdv2/pages/modules/indsnp/indsnp_search.jsp.
this is the code that I have for now.
driver = webdriver.Chrome(executable_path=path_to_chromebrowser)
driver.get("https://bigd.big.ac.cn/dogsdv2/pages/modules/indsnp/indsnp_search.jsp")
input_area = driver.find_element_by_name("searchForm.genename")
input_area.send_keys("P2RY12")
searcher = driver.find_element_by_class_name("button")
searcher.click()
# table = driver.find_element_by_class_name("table7 table7-border")
# table.find_element_by_tag_name("a").click()
I am trying to click the first SNP ID that comes up, upon search. What would be the good way for me to click the href of the search result?
ON the webpage https://bigd.big.ac.cn/dogsdv2/pages/modules/indsnp/indsnp_search.jsp to search for the Gene Name as P2RY12 and click the first SNP ID that comes up upon search you need to induce WebDriverWait for the element_to_be_clickable() and you can use the following Locator Strategies:
Code Block:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = webdriver.ChromeOptions()
options.add_argument("start-maximized")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
driver = webdriver.Chrome(options=options, executable_path=r'C:\WebDrivers\chromedriver.exe')
driver.get('https://bigd.big.ac.cn/dogsdv2/pages/modules/indsnp/indsnp_search.jsp')
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "input#idgname[name='searchForm.genename']"))).send_keys("P2RY12")
driver.find_element_by_css_selector("button.button[type='submit']").click()
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "form[action^='/dogsdv2/com/exportFile'] table>tbody>tr td:nth-child(3)>a"))).click()
Browser Snapshot:
Try this:
firstsnpID = driver.find_element_by_xpath("(.//table[#class='table7 table7-border']/tbody/tr/td[3]/a)[1]")
firstsnpID.click()
you can not use compound classes to locate element using find_element_by_class_name
driver.find_element_by_xpath('/html/body/div/div[2]/div[2]/form/table/tbody/tr[1]/td[3]/a[1]').click()
If you need other ids:
for id in range(1,10):
driver.find_element_by_xpath('/html/body/div/div[2]/div[2]/form/table/tbody/tr[{}]/td[3]/a[1]'.format(id)).click()
sleep(5)
driver.back()
To click on first link on the table induce WebDriverWait() and element_to_be_clickable() and following CSS selector.
Code:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Chrome(executable_path=path_to_chromebrowser)
driver.get("https://bigd.big.ac.cn/dogsdv2/pages/modules/indsnp/indsnp_search.jsp")
input_area = driver.find_element_by_name("searchForm.genename")
input_area.send_keys("P2RY12")
searcher = driver.find_element_by_class_name("button")
searcher.click()
WebDriverWait(driver,10).until(EC.element_to_be_clickable((By.CSS_SELECTOR,"table.table7.table7-border td>a[href^='/dogsdv2/refsnp/showRefSNPDetail']"))).click()
To get all the link induce WebDriverWait() and visibility_of_all_elements_located() and get the href value then iterate each url
allelemets=WebDriverWait(driver,10).until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR,"table.table7.table7-border td>a[href^='/dogsdv2/refsnp/showRefSNPDetail']")))
allurls=[item.get_attribute('href') for item in allelemets]
print(allurls)
for link in allurls:
driver.get(link)
I am trying to scrape this:
https://www.lanebryant.com/chiffon-faux-wrap-fit-flare-midi-dress/prd-355958#color/0000091393
And this is my code:
wait = WebDriverWait(d, 10)
close = wait.until(EC.element_to_be_clickable((By.XPATH, "//a[#id='closeButton']")))
close.click()
time.sleep(5)
chart = wait.until(EC.element_to_be_clickable((By.XPATH, "//div[contains(*,'Size Guide')][#class='size-chart-link']")))
chart.click()
It first closes the pop up and then clicks the size guide, However, it always gives timeout exception and works only a couple of times.
The PARTIAL_LINK_TEXT Size Guide is pretty much unique within the page so would be your best bet would be to:
Induce WebDriverWait for invisibility_of_element() for the wrapper element
Induce WebDriverWait for the element_to_be_clickable() for the desired element
You can use the following Locator Strategy:
Code Block (using XPATH and PARTIAL_LINK_TEXT):
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = webdriver.ChromeOptions()
options.add_argument('start-maximized')
driver = webdriver.Chrome(options=options, executable_path=r'C:\WebDrivers\chromedriver.exe')
driver.get('https://www.lanebryant.com/chiffon-faux-wrap-fit-flare-midi-dress/prd-355958#color/0000091393')
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//a[#id='closeButton']"))).click()
WebDriverWait(driver, 20).until(EC.invisibility_of_element((By.XPATH, "//div[#id='tinymask']")))
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.PARTIAL_LINK_TEXT, "Size Guide"))).click()
Code Block (using CSS_SELECTOR and PARTIAL_LINK_TEXT):
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = webdriver.ChromeOptions()
options.add_argument('start-maximized')
driver = webdriver.Chrome(options=options, executable_path=r'C:\WebDrivers\chromedriver.exe')
driver.get('https://www.lanebryant.com/chiffon-faux-wrap-fit-flare-midi-dress/prd-355958#color/0000091393')
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "a#closeButton"))).click()
WebDriverWait(driver, 20).until(EC.invisibility_of_element((By.CSS_SELECTOR, "div#tinymask")))
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.PARTIAL_LINK_TEXT, "Size Guide"))).click()
Browser Snapshot:
Use JavaScript Executor to click on the element.Seems like selenium webdriver unable to click on the element.Use the below xpath
d.get("https://www.lanebryant.com/chiffon-faux-wrap-fit-flare-midi-dress/prd-355958#color/0000091393")
wait = WebDriverWait(d, 10)
close = wait.until(EC.element_to_be_clickable((By.XPATH, "//a[#id='closeButton']")))
close.click()
chart = wait.until(EC.element_to_be_clickable((By.XPATH, "//div[#class='size-chart-link']/a[contains(.,'Size Guide')]")))
d.execute_script("arguments[0].click();", chart)
Browser snapshot: