Headless Chrome does not load the page and gets stuck at:
wait = WebDriverWait(driver, 30)
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, ".price-link:nth-child(1) .team-name")))
Why is it doing this? Is this a bug as it works perfectly in normal chrome and prints h below except in headless...
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
options = Options()
options.add_argument("--headless")
options.add_argument("window-size=1400,600")
driver = webdriver.Chrome(chrome_options=options)
#driver = webdriver.Chrome()
driver.set_window_size(1024, 600)
driver.maximize_window()
driver.get('https://www.sportsbet.com.au/betting/soccer?LeftNav')
print('?')
wait = WebDriverWait(driver, 30)
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, ".price-link:nth-child(1) .team-name")))
print('h')
Another example:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
options = Options()
options.add_argument("--headless")
options.add_argument("window-size=1400,600")
driver = webdriver.Chrome(chrome_options=options)
#driver = webdriver.Chrome()
driver.set_window_size(1024, 600)
driver.maximize_window()
import time
driver.get('https://www.sportsbet.com.au/betting/soccer?LeftNav')
import time
time.sleep(10)
langs = driver.find_elements_by_css_selector(".price-link:nth-child(1) .team-name")
langs_text = []
for lang in langs:
print(lang.text)
langs_text.append(lang.text)
print('h')
Job prints with removed wait until though nothing is scraped. Page not loading is likely issue.
Try to use the following code:
element = WebDriverWait(driver, 30).until(EC.visibility_of_element_located((By.CSS_SELECTOR, ".price-link:nth-child(1) .team-name")))
driver.execute_script("arguments[0].scrollIntoView(true);", element)
Hope it helps you!
Related
This is my first attempt to login a website using selenium, i have written the below piece of code but not able to login the website, getting the following error message.
Message: no such element: Unable to locate element: {"method":"css selector","selector":"[name="txtUsername"]"}
(Session info: chrome=108.0.5359.95)
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
s=Service("C:\Program Files\Google\Chrome\Application\chromedriver.exe")
driver = webdriver.Chrome(service=s)
driver.maximize_window()
driver.get("https://opensource-demo.orangehrmlive.com")
wait = WebDriverWait(driver, 10)
driver.find_element(By.NAME,"txtUsername").send_keys("Admin")
driver.find_element(By.ID, "txtPassword").send_keys("admin123")
driver.quit()
Looking for some suggestions
There are 2 problems with your code:
You created the wait object but not used it...
Your locators are wrong.
The following code works:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(service=webdriver_service, options=options)
wait = WebDriverWait(driver, 20)
url = "https://opensource-demo.orangehrmlive.com"
driver.get(url)
wait.until(EC.element_to_be_clickable((By.NAME, "username"))).send_keys("Admin")
wait.until(EC.element_to_be_clickable((By.NAME, "password"))).send_keys("admin123")
wait.until(EC.element_to_be_clickable((By.TAG_NAME, "button"))).click()
The wait needs to be called:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
s=Service("C:\Program Files\Google\Chrome\Application\chromedriver.exe")
driver = webdriver.Chrome(service=s)
driver.maximize_window()
driver.get("https://opensource-demo.orangehrmlive.com")
wait = WebDriverWait(driver, 10)
user = wait.until(EC.element_to_be_clickable((By.XPATH, "//div")))
user.send_keys("Admin")
driver.find_element(By.ID, "txtPassword").send_keys("admin123")
driver.quit()
I want to handle the progress bar to be stopped after a certain percent let's say 70%. So far I have got the solutions, they all are using .attributeToBe() method. But in selenium 4 I don't have that method present. How can I handle that?
This is the demo link - https://demoqa.com/progress-bar
I have tried to do that using explicit wait like others but I didn't find .attributrToBe() method in selenium 4. Is it possible to do that using loop?
You can use text_to_be_present_in_element_attribute expected_conditions.
The following code works:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(options=options, service=webdriver_service)
wait = WebDriverWait(driver, 60)
url = "https://demoqa.com/progress-bar"
driver.get(url)
wait.until(EC.element_to_be_clickable((By.ID, "startStopButton"))).click()
wait.until(EC.text_to_be_present_in_element_attribute((By.CSS_SELECTOR, '[role="progressbar"]'),"aria-valuenow","70"))
wait.until(EC.element_to_be_clickable((By.ID, "startStopButton"))).click()
UPD
For the second page the following code works as well:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(options=options, service=webdriver_service)
wait = WebDriverWait(driver, 60)
url = "http://uitestingplayground.com/progressbar"
driver.get(url)
wait.until(EC.element_to_be_clickable((By.ID, "startButton"))).click()
wait.until(EC.text_to_be_present_in_element_attribute((By.CSS_SELECTOR, '[role="progressbar"]'),"aria-valuenow","75"))
wait.until(EC.element_to_be_clickable((By.ID, "stopButton"))).click()
I am not getting price they give me empty output this is page link https://www.amazon.com/dp/B00M0DWQYI?th=1
from selenium import webdriver
import time
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support.select import Select
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
url='https://www.amazon.com/dp/B00M0DWQYI?th=1'
PATH="C:\Program Files (x86)\chromedriver.exe"
driver =webdriver.Chrome(PATH)
driver.get(url)
item=dict()
try:
item['price'] = driver.find_element(By.XPATH, "//div[#id='corePrice_feature_div'] //span[#class='a-offscreen']").text
except:
item['price']=''
print(item)
You may want to wait for that element to properly load, prior to locating it:
[...]
wait = WebDriverWait(driver, 10)
item['price'] = wait.until(EC.element_to_be_clickable((By.XPATH, "//div[#id='corePrice_feature_div']//span[#class='a-offscreen']"))).text
Selenium documentation can be found at https://www.selenium.dev/documentation/
EDIT: Here is a complete example of how you can get that information:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
import time as t
chrome_options = Options()
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument('disable-notifications')
chrome_options.add_argument("window-size=1920,1080")
webdriver_service = Service("chromedriver/chromedriver") ## path to where you saved chromedriver binary
driver = webdriver.Chrome(service=webdriver_service, options=chrome_options)
wait = WebDriverWait(driver, 5)
items = dict()
driver.get('https://www.amazon.com/dp/B00M0DWQYI?th=1')
t.sleep(1)
driver.refresh()
items['price'] = wait.until(EC.element_to_be_clickable((By.XPATH, '//div[#id="corePrice_feature_div"]//span[#class="a-price aok-align-center"]'))).text.replace('\n', '.')
print(items)
Result in terminal:
{'price': '$32.98'}
You need to wait for element visibility and then to extract it's text.
The following Selenium code works:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(service=webdriver_service, options=options)
url = 'https://www.amazon.com/dp/B00M0DWQYI'
driver.get(url)
wait = WebDriverWait(driver, 10)
print(wait.until(EC.visibility_of_element_located((By.XPATH, "//div[#id='corePrice_feature_div']"))).text)
The output is
$32
98
You can use bs4 and it will work fine
from bs4 import BeautifulSoup
soup = BeautifulSoup(driver.page_source, 'lxml')
try:
item['price'] = soup.find('input', id="attach-base-product-price").get('value')
except:
item['price'] = ''
finally:
driver.close()
driver.quit()
print(item)
https://www.espncricinfo.com/player/aamer-jamal-793441
This is the URL and here i am trying to access Full Name "Aamer Jamal". with the help of selenium web driver. But I dont know why it gives
NoSuchElementException
`the code is written below:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium_firefox import Firefox
import time
import pandas as pd
driver = webdriver.Firefox()
#Reach to the Landing page
driver.get('https://www.espncricinfo.com/player/aamer-jamal-793441')
driver.maximize_window()
time.sleep(25)
not_now = driver.find_element(By.ID, 'wzrk-cancel')
not_now.click()
fullname = driver.find_element(By.XPATH, '/html/body/div[1]/section/section/div[4]/div[2]/div/div[1]/div/div/div[1]/div[1]/span/h5')
print(fullname.text)`
Error :
NoSuchElementException: Message: Unable to locate element: /html/body/div[1]/section/section/div[4]/div[2]/div/div[1]/div/div/div[1]/div[1]/span
You have to use WebDriverWait expected_conditions explicit waits, not a long hardcoded pauses. You also have to learn how to create correct locators. Long absolute XPaths and CSS Selectors are extremely breakable. The following code works:
from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(options=options, service=webdriver_service)
wait = WebDriverWait(driver, 60)
actions = ActionChains(driver)
url = "https://www.espncricinfo.com/player/aamer-jamal-793441"
driver.get(url)
wait.until(EC.element_to_be_clickable((By.ID, 'wzrk-cancel')))
fullname = wait.until(EC.visibility_of_element_located((By.CLASS_NAME, 'ds-text-title-l'))).text
print(fullname)
The output is:
Aamer Jamal
Hello I'm newbie of selenium.
and I got the problem. I want to scrap using XPATH and not working.
from __future__ import print_function
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
driver = webdriver.Chrome('chromedriver', options=chrome_options)
driver.set_window_size(1320, 550)
exchange_link = "https://icodrops.com/category/ended-ico/"
driver.get(exchange_link)
wait = WebDriverWait(driver, 30)
ico_links = driver.find_element(By.XPATH, "//div[#id='all']//div[#id='ajaxc']")
But the result is this
NoSuchElementException: Message: no such element: Unable to locate element: {"method":"xpath","selector":"//div[#id='all']//div[#id='ajaxc']"}(Session info: headless chrome=97.0.4692.71)
I don`t know why it cant be work.
I just did by XPath.
image description here
my code link is this,
Scrap page is this
You are missing a wait / delay.
You can use expected conditions to wait for that element presence.
As following:
from __future__ import print_function
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
driver = webdriver.Chrome('chromedriver', options=chrome_options)
driver.set_window_size(1320, 550)
exchange_link = "https://icodrops.com/category/ended-ico/"
driver.get(exchange_link)
wait = WebDriverWait(driver, 30)
wait.until(EC.visibility_of_element_located((By.XPATH, "//h3/a[#rel]")))
time.sleep(1)
ico_links= driver.find_element_by_xpath('//h3/a[#rel]')