I am trying to get the attribute "onclick" (onclick="openDirection( 41.051777,28.986551)") of the first button element inside all of the li elements. I want to print all the coordinates. In the code above I can manage to print the first two element's coordinates. However, a NoSuchElementException is thrown afterwards.
driver.get('https://migroskurumsal.com/magazalarimiz/')
try:
select = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.ID, 'stores'))
)
print('Dropdown is ready!')
except TimeoutException:
print('Took too much time!')
select = Select(driver.find_element(By.ID, 'stores'))
select.select_by_value('2')
shopList = driver.find_element(By.ID, "shopList")
try:
WebDriverWait(driver, 10).until(
EC.visibility_of_all_elements_located((By.XPATH, "//ul[#id='shopList']/li/div/button"))
)
print('Shoplist is ready!')
except TimeoutException:
print('Took too much time!')
driver.quit()
print(shopList.get_attribute("class"))
li_items = shopList.find_elements(By.TAG_NAME, 'li')
for item in li_items:
first = item.find_element(By.TAG_NAME, 'div')
second = first.find_element(By.TAG_NAME, 'button')
coord = second.get_attribute("onclick")
print(coord)
There are several issues here:
You are using wrong locators
You will need to scroll the page in order to access the elements initially not in the visible view.
This should work better:
from selenium.webdriver.common.action_chains import ActionChains
driver.get('https://migroskurumsal.com/magazalarimiz/')
actions = ActionChains(driver)
try:
select = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, 'stores')))
print('Dropdown is ready!')
except TimeoutException:
print('Took too much time!')
select = Select(driver.find_element(By.ID, 'stores'))
select.select_by_value('2')
shopList = driver.find_element(By.ID, "shopList")
try:
WebDriverWait(driver, 10).until(EC.visibility_of_all_elements_located((By.XPATH, "//ul[#id='shopList']/li/div/button")))
time.sleep(1)
print('Shoplist is ready!')
except TimeoutException:
print('Took too much time!')
driver.quit()
li_items = shopList.find_elements(By.CSS_SELECTOR, 'li.m-bg-cream')
for idx, val in enumerate(li_items):
li_items = shopList.find_elements(By.CSS_SELECTOR, 'li.m-bg-cream')
item = li_items[idx]
actions.move_to_element(item).perform()
time.sleep(0.5)
button = item..find_elements(By.CSS_SELECTOR, "button[onclick^='set']")
coord = button.get_attribute("onclick")
print(coord)
Related
My python script to browse a website in headless mode using selenium was working fine for a long time but it stopped working suddenly. 'Submit' button on [this][1] page was clickable easily but not anymore.
This is part of my code which is not working anymore:
def result(update: Update, context: CallbackContext):
global stop_alert, chat_id
upd = bot.get_updates()
chat_id = upd[0].message.from_user.id
try:
if platform.startswith('win'):
ser = Service("C:\\Program Files (x86)\\chromedriver.exe")
op = webdriver.ChromeOptions()
op.headless = True
driver = webdriver.Chrome(service=ser, options=op)
else:
ser = Service("/usr/local/bin/chromedriver")
op = webdriver.ChromeOptions()
op.headless = True
op.add_argument('--no-sandbox')
driver = webdriver.Chrome(service=ser, options=op)
except Exception as e:
update.message.reply_text(f'<b>Exception occurred in result function\n'
f'while creating driver object</b>\n'
f' {str(e)}', parse_mode='HTML')
return
reply = f"\U00002764<b> Welcome to Results on BSE Bot </b>\U00002764\n" \
f"🕵 {len(companies)} company(s) to scan 🕵"
update.message.reply_text(reply, parse_mode='HTML')
channel_msg_id = ''
wait = WebDriverWait(driver, 10)
while True:
if not stop_alert:
if companies:
for i in list(companies):
try:
driver.get(i)
wait.until(
EC.presence_of_element_located((By.CSS_SELECTOR, "div.whitebox img[src*='loader']")))
wait.until_not(
EC.presence_of_element_located((By.CSS_SELECTOR, "div.whitebox img[src*='loader']")))
select = Select(driver.find_element(By.ID, 'ddlPeriod'))
select.select_by_value('Result')
driver.find_element(By.XPATH, '//input[#value="Submit"]').click()
table_data = WebDriverWait(driver, 30).until(
EC.presence_of_element_located((By.XPATH, "//td[#id='lblann']")))
except Exception as e:
reply = f"\U00002764 Exception occurred. {str(e)} \U00002764"
update.message.reply_text(reply, parse_mode='HTML')
companies.remove(i)
continue
try:
exch_rec_date = table_data.find_element(
By.XPATH, '//*[#id="lblann"]/table/tbody/tr[4]/td/table[1]/tbody/tr[2]/td/b[1]'). \
get_attribute("innerHTML")
exch_rec_date_modified = datetime.strptime(exch_rec_date, ' %d-%m-%Y %H:%M:%S').date()
company_name = table_data.find_element(
By.XPATH, '//*[#id="lblann"]/table/tbody/tr[1]/td/a').get_attribute("innerHTML")
heading = table_data.find_element(
By.XPATH, '//*[#id="lblann"]/table/tbody/tr[4]/td/table[1]/tbody/tr[1]/td[1]/a'). \
get_attribute("innerHTML")
link_pdf = table_data.find_element(By.XPATH, "//a[#class='tablebluelink']").get_attribute(
'href')
exch_rec_time = table_data.find_element(
By.XPATH, '//*[#id="lblann"]/table/tbody/tr[4]/td/table[1]/tbody/tr[2]/td/b[1]'). \
get_attribute("innerHTML")
This is the error message:
Message: element click intercepted: Element is not clickable at point (217, 1879)
(Session info: headless chrome=107.0.5304.107)
Stacktrace:
Backtrace:
Ordinal0 [0x00471ED3+2236115]
Ordinal0 [0x004092F1+1807089]
[1]: https://www.bseindia.com/stock-share-price/tanla-platforms-ltd/tanla/532790/corp-announcements/
You have to wait for the elements to be clickable.
First of all this related to the submit button you are clicking, but I'd add this check also before selecting the Result since I see when the page is opened spinner is appeared until the page is loaded.
So, I'd change your code to be as following:
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
wait = WebDriverWait(driver, 20)
for i in list(companies):
try:
driver.get(i)
wait.until_not(EC.presence_of_element_located((By.CSS_SELECTOR, "div.whitebox img[src*='loader']")))
select = Select(driver.find_element(By.ID, 'ddlPeriod'))
select.select_by_value('Result')
wait.until(EC.element_to_be_clickable((By.XPATH, '//input[#value="Submit"]'))).click()
table_data = WebDriverWait(driver, 30).until(
EC.presence_of_element_located((By.XPATH, "//td[#id='lblann']")))
except Exception as e:
reply = f"\U00002764 Exception occurred. {str(e)} \U00002764"
update.message.reply_text(reply, parse_mode='HTML')
companies.remove(i)
continue
I am trying to get li items in a ul. Here is my code:
driver.get('https://migroskurumsal.com/magazalarimiz/')
try:
select = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.ID, 'stores'))
)
print('Dropdown is ready!')
except TimeoutException:
print('Took too much time!')
select = Select(driver.find_element(By.ID,'stores'))
select.select_by_value('2')
try:
shopList = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.ID, "shopList"))
)
print('Shoplist is ready!')
except TimeoutException:
print('Took too much time!')
driver.quit()
print(shopList.get_attribute("class"))
li_items = shopList.find_elements(By.TAG_NAME,'li')
print(len(li_items))
I located the ul element with id=shopList successfully. Then I tried to get all of the li elements under ul by using find_elements(By.TAG_NAME). I also tried find_elements(By.CLASS_NAME), however the len(li_items) is always 0. I kindly request your help. Thank you.
In your case waiting until EC.presence_of_element_located((By.ID, "shopList")) does not do the trick, as this ul is already on the page, even before selecting filters, it's just empty. Instead, you could wait until li child elements are visible.
Example:
driver.get('https://migroskurumsal.com/magazalarimiz/')
try:
select = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.ID, 'stores'))
)
print('Dropdown is ready!')
except TimeoutException:
print('Took too much time!')
select = Select(driver.find_element(By.ID, 'stores'))
select.select_by_value('2')
shopList = driver.find_element(By.ID, "shopList")
try:
WebDriverWait(driver, 10).until(
EC.visibility_of_all_elements_located((By.XPATH, "//ul[#id='shopList']/li"))
)
print('Shoplist is ready!')
except TimeoutException:
print('Took too much time!')
driver.quit()
print(shopList.get_attribute("class"))
li_items = shopList.find_elements(By.TAG_NAME, 'li')
print(len(li_items))
Results with output:
Dropdown is ready!
Shoplist is ready!
shop-list list-unstyled
601
I have an e-commerce page and there are multiple products on a page. I need to click the link of a product then return on the main page and click the link of the next product, but when I return, the elements can't be found anymore.
Path = "C:\Program Files (x86)\chromedriver.exe"
driver = webdriver.Chrome(Path)
driver.get("https://www.emag.ro/")
search_bar = driver.find_element_by_id("searchboxTrigger")
search_bar.send_keys("laptopuri")
search_bar.send_keys(Keys.RETURN)
main = None
try:
main = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.ID, "main-container"))
)
print("Page loaded,main retrived succesfully")
print(" ")
except:
driver.quit()
products = main.find_elements_by_css_selector("div.card-item.js-product-data")
for product in products:
raw_name = product.text
raw_price = product.find_element_by_css_selector("p.product-new-price").text
link = product.find_element_by_tag_name("a")
#clicking the link
link.click()
spec_page = None
try:
spec_page = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.CLASS_NAME, "col-md-12"))
)
except:
driver.quit()
print(spec_page)
driver.back()
After the first iteration, I get the following error :
selenium.common.exceptions.StaleElementReferenceException: Message: stale element reference: element is not attached to the page document on line raw_name = product.text,basically at the beginning of the loop.
I assume the page is not loading properly or something like that, I tried using time.sleep before going through the loop but nothing
When you are writing driver.back(), it will go back to previous page and by the time it will reach to original page all the defined elements will become stale. You need to redefined them like below :-
This should handle the exception.
products = len(main.find_elements_by_css_selector("div.card-item.js-product-data"))
j = 0
for product in range(products):
elements = main.find_elements_by_css_selector("div.card-item.js-product-data")
raw_name = elements[j].text
raw_price = elements[j].find_element_by_css_selector("p.product-new-price").text
link = elements[j].find_element_by_tag_name("a")
# clicking the link
link.click()
spec_page = None
try:
spec_page = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.CLASS_NAME, "col-md-12"))
)
except:
driver.quit()
print(spec_page)
j = j + 1
driver.back()
mail = driver.find_element_by_name('identifier')
mail.clear()
mail.send_keys('example#gmail.com')
mail_next = driver.find_element_by_id('identifierNext')
mail_next.click()
time.sleep(2)
password = driver.find_element_by_name('password')
password.clear()
password.send_keys('102938')
password_next = driver.find_element_by_id('passwordNext')
password_next.click()
time.sleep(2)
compose = driver.find_element_by_class_name('T-I J-J5-Ji T-I-KE L3')
compose.click()
time.sleep(2)
to = driver.find_element_by_name('to')
to.clear()
to.send_keys('example#gmail.com')
subject = driver.find_element_by_name('subjectbox')
subject.clear()
subject.send_keys('Subject')
body = driver.find_element_by_id('Am Al editable LW-avf tS-tW')
body.clear()
body.send_keys('bro, it works')
The error is "Message: Unable to locate element: .T-I J-J5-Ji T-I-KE L3"
I even tried Xpath, so it is clear that the class isn't present in the driver
I couldn't find out what I'm doing wrong here
Try the following to click the compose button in GMAIL
try:
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.XPATH, "//div[contains(text(),'Compose')]"))
)
except:
driver.quit()
compose = driver.find_element_by_xpath("//div[contains(text(),'Compose')]")
compose.click()
If you want to do it by class, the following code also works.
try:
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.XPATH, "//div[#class = 'T-I J-J5-Ji T-I-KE L3']"))
)
except:
driver.quit()
compose = driver.find_element_by_xpath("//div[#class = 'T-I J-J5-Ji T-I-KE L3']")
compose.click()
I am trying to scrape the reviews of a hotel from TripAdvisor and write it in a text file. So far the code is doing good except the fact that every now and then it throws StaleElementException on the line where I am writing the text file. Here's is my code:
for num in range(page_count):
try:
if num != 0:
try:
nxt = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, "a.nav.next.rndBtn.ui_button.primary.taLnk")))
#nxt = driver.find_element_by_css_selector("a.nav.next.rndBtn.ui_button.primary.taLnk")
nxt.click()
driver.implicitly_wait(5)
except NoSuchElementException:
driver.refresh()
#driver.implicitly_wait(5)
nxt = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, "a.nav.next.rndBtn.ui_button.primary.taLnk")))
nxt.click()
driver.implicitly_wait(5)
try:
more = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "span.taLnk.ulBlueLinks")))
#more = driver.find_element_by_css_selector("span.taLnk.ulBlueLinks")
more.click()
time.sleep(1)
except TimeoutException:
print("There is no 'more' button on page %d" % (num+1))
except WebDriverException:
nxt = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, "span.taLnk.ulBlueLinks")))
nxt.click()
driver.implicitly_wait(5)
review_result = WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.CLASS_NAME, 'entry')))
with open('New_Review.txt', 'a') as fid:
for review in review_result:
fid.write(unidecode(review.text))
fid.write(sep)
fid.write(line_break)
print ("processing done till page number %d" % (num+1))
except StaleElementReferenceException:
driver.refresh()
driver.implicitly_wait(5)
try:
more = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "span.taLnk.ulBlueLinks")))
#more = driver.find_element_by_css_selector("span.taLnk.ulBlueLinks")
more.click()
except TimeoutException:
print("There is no 'more' button on page %d" % (num+1))
except WebDriverException:
nxt = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, "span.taLnk.ulBlueLinks")))
nxt.click()
driver.implicitly_wait(5)
review_result = WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.CLASS_NAME, 'entry')))
#print (review_result[1].text)
with open('New_Review.csv', 'a') as fid:
writer = csv.writer(fid, delimiter = ',', line_break = '\n')
for review in review_result:
fid.write(unidecode(review.text))
fid.write(sep)
fid.write(line_break)
print ("processing done till page number %d" % (num+1))
Here is the error:
StaleElementReferenceException: stale element reference: element is
not attached to the page document
The traceback gives this line:
fid.write(unidecode(review.text))
I have already tried to handle the exception but its not working for me and I am having a hard time trying to figure out where exactly am I wrong. Any help is appreciated!
Try creating helper method such as
def get_text(locator):
staled = True
while staled:
try:
return WebDriverWait(driver, 10).until(EC.presence_of_element(locator)).text
except StaleElementReferenceException:
`log something or limit retry to certain times`
then change how you get the text
review_result = WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located())
num_of_review - review_result.length
with open('New_Review.txt', 'a') as fid:
for index in range(1, num_of_review):
review_text = get_text((By.XPATH, "//*[#class='entry'][index]"))
fid.write(unidecode(review_text))
fid.write(sep)
fid.write(line_break)