I'm trying to extract 1000 usernames and the team rosters for each username in a list from this SPA (single page application) sports website (requires free login): https://rumble.otmnft.com/contest/live?contest=Rumble
I've tried numerous strategies to try to avoid the common StaleElementReferenceException problem in Selenium including try / except block, WebDriverWait and time.sleep(5) among many others, but none have worked. Would be amazing if someone knows an alternative solution which could work as I've been trying to solve this by myself for quite a few days now.
Here is the relevant part of the latest attempted code which still throws StaleElementReferenceException below:
parent_dict = {}
parent_dict['data'] = []
USERNAME_XPATH = "//span[#class='hidden lg:flex font-semibold']"
ROSTER_XPATH = "//span[#class='text-white hidden lg:flex font-semibold']"
WebDriverWait(driver, 50).until(EC.presence_of_element_located((By.XPATH, "//table")))
username_elements = driver.find_elements(By.XPATH, USERNAME_XPATH)
try:
for usr_element in username_elements:
usr_element.click()
except StaleElementReferenceException:
# Find the element again and continue with the loop
WebDriverWait(driver, 50).until(EC.presence_of_element_located((By.XPATH, USERNAME_XPATH)))
username_elements = driver.find_elements(By.XPATH, USERNAME_XPATH)
time.sleep(5)
for usr_element in username_elements:
usr_element.click()
WebDriverWait(driver, 50).until(EC.presence_of_element_located((By.XPATH, ROSTER_XPATH)))
roster_elements = driver.find_elements(By.XPATH, ROSTER_XPATH)
roster = [cell.text for cell in roster_elements]
TITLE_USER_XPATH = "//div[#class='flex flex-col space-y-2']/span"
title_user = driver.find_element(By.XPATH, TITLE_USER_XPATH)
child_dict = {"username": title_user.text, "roster": roster }
print("CHILD_DICT", child_dict)
parent_dict['data'].append(child_dict)
with open('data_v2.json', 'w') as outfile:
json.dump(parent_dict, outfile)
print("PARENT_DICT", parent_dict)
return parent_dict
In the browser I can see that the script is able to click on the very first username but it fails when trying click on the second username, but reports the error for the line usr_element.click() both inside the try and except blocks
Full error trace is:
Traceback (most recent call last):
File "/Users/danpro12/Dropbox/01 FREE TIME STUFF/01 WEB DEVELOPMENT/12 DAPPER RELATED/otm_python_scraper/otm_specific/utils.py", line 23, in sub_func
usr_element.click()
File "/Users/danpro12/.virtualenvs/otm/lib/python3.9/site-packages/selenium/webdriver/remote/webelement.py", line 88, in click
self._execute(Command.CLICK_ELEMENT)
File "/Users/danpro12/.virtualenvs/otm/lib/python3.9/site-packages/selenium/webdriver/remote/webelement.py", line 396, in _execute
return self._parent.execute(command, params)
File "/Users/danpro12/.virtualenvs/otm/lib/python3.9/site-packages/selenium/webdriver/remote/webdriver.py", line 429, in execute
self.error_handler.check_response(response)
File "/Users/danpro12/.virtualenvs/otm/lib/python3.9/site-packages/selenium/webdriver/remote/errorhandler.py", line 243, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.StaleElementReferenceException: Message: The element reference of <span class="hidden lg:flex font-semibold"> is stale; either the element is no longer attached to the DOM, it is not in the current frame context, or the document has been refreshed
Stacktrace:
RemoteError#chrome://remote/content/shared/RemoteError.sys.mjs:8:8
WebDriverError#chrome://remote/content/shared/webdriver/Errors.sys.mjs:180:5
StaleElementReferenceError#chrome://remote/content/shared/webdriver/Errors.sys.mjs:461:5
element.resolveElement#chrome://remote/content/marionette/element.sys.mjs:674:11
evaluate.fromJSON#chrome://remote/content/marionette/evaluate.sys.mjs:255:31
evaluate.fromJSON#chrome://remote/content/marionette/evaluate.sys.mjs:263:29
receiveMessage#chrome://remote/content/marionette/actors/MarionetteCommandsChild.sys.mjs:74:34
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/danpro12/Dropbox/01 FREE TIME STUFF/01 WEB DEVELOPMENT/12 DAPPER RELATED/otm_python_scraper/otm_specific/rumble_client.py", line 53, in <module>
sub_func(driver)
File "/Users/danpro12/Dropbox/01 FREE TIME STUFF/01 WEB DEVELOPMENT/12 DAPPER RELATED/otm_python_scraper/otm_specific/utils.py", line 30, in sub_func
usr_element.click()
File "/Users/danpro12/.virtualenvs/otm/lib/python3.9/site-packages/selenium/webdriver/remote/webelement.py", line 88, in click
self._execute(Command.CLICK_ELEMENT)
File "/Users/danpro12/.virtualenvs/otm/lib/python3.9/site-packages/selenium/webdriver/remote/webelement.py", line 396, in _execute
return self._parent.execute(command, params)
File "/Users/danpro12/.virtualenvs/otm/lib/python3.9/site-packages/selenium/webdriver/remote/webdriver.py", line 429, in execute
self.error_handler.check_response(response)
File "/Users/danpro12/.virtualenvs/otm/lib/python3.9/site-packages/selenium/webdriver/remote/errorhandler.py", line 243, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.StaleElementReferenceException: Message: The element reference of <span class="hidden lg:flex font-semibold"> is stale; either the element is no longer attached to the DOM, it is not in the current frame context, or the document has been refreshed
Stacktrace:
RemoteError#chrome://remote/content/shared/RemoteError.sys.mjs:8:8
WebDriverError#chrome://remote/content/shared/webdriver/Errors.sys.mjs:180:5
StaleElementReferenceError#chrome://remote/content/shared/webdriver/Errors.sys.mjs:461:5
element.resolveElement#chrome://remote/content/marionette/element.sys.mjs:674:11
evaluate.fromJSON#chrome://remote/content/marionette/evaluate.sys.mjs:255:31
evaluate.fromJSON#chrome://remote/content/marionette/evaluate.sys.mjs:263:29
receiveMessage#chrome://remote/content/marionette/actors/MarionetteCommandsChild.sys.mjs:74:34
Seems that after you do usr_element.click() DOM updated and so you cannot iterate over username_elements since all elements in this list are stale.
Try this solution
username_elements_length = len(driver.find_elements(By.XPATH, USERNAME_XPATH))
for usr_element_index in range(username_elements_length):
username_element = driver.find_elements(By.XPATH, USERNAME_XPATH)[usr_element_index]
username_element.click()
WebDriverWait(driver, 5).until(EC.staleness_of(username_element))
Related
I want to have Selenium wait until a webpage is loaded. I use Safari 14.1.2 on macOS 10.15.7
So I tried as below:
// login process and it successes and the following page shows up
login_submit.click()
// the page after the authorization, make selenium wait until ```hoge``` is clickable
WebDriverWait(browser, 10).until(EC.element_to_be_clickable((By.ID, 'hoge')))
This code does not work as expected. It fails with an error selenium.common.exceptions.NoSuchFrameException:. The error comes immediately, which seems there is no 10 s wait.
The stack trace is as below: (Paths are modified.)
Traceback (most recent call last):
File "/login.py", line 99, in <module>
login_schedule().quit()
File "/login.py", line 67, in login_schedule
wait.until(EC.presence_of_element_located((By.ID, 'HOGEHOGE')))
File "/selenium/webdriver/support/wait.py", line 71, in until
value = method(self._driver)
File "/selenium/webdriver/support/expected_conditions.py", line 64, in __call__
return _find_element(driver, self.locator)
File "/selenium/webdriver/support/expected_conditions.py", line 415, in _find_element
raise e
File "/selenium/webdriver/support/expected_conditions.py", line 411, in _find_element
return driver.find_element(*by)
File "/selenium/webdriver/remote/webdriver.py", line 976, in find_element
return self.execute(Command.FIND_ELEMENT, {
File "/selenium/webdriver/remote/webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "/selenium/webdriver/remote/errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchFrameException: Message:
When I put time.sleep(10) instead as below,
import time
// login process and it successes and the following page shows up
login_submit.click()
time.sleep(10)
// the page after the authorization, make selenium wait until ```hoge``` is clickable
WebDriverWait(browser, 10).until(EC.element_to_be_clickable((By.ID, 'hoge')))
it works as expected.
Note that the last part can even be replaced
browser.find_element_by_xpath("//select[#id='hoge']")
What is wrong with my Python code?
try to use this function to wait until the element or something appears:
def wait_for(xpath):
while True:
try:
driver.find_element_by_xpath(xpath)
return True
except NoSuchElementException:
continue
do not forget
from selenium.common.exceptions import NoSuchElementException
you can use anything it is not important to be XPATH as a method.
I am trying to iterate over pages having a program List and then I have to select checkbox defined in an image below in order to get the run length of a loop on-page.
But when I am trying to navigate over pages via the next button it will stop clicking on checkboxes and throws me below error.
I have tried with xpath,css selector and id method But all get failed.
DOM element SS along with checkbox highlighted
Line of Code Used-
CheckBoxclick = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "#workflow_table > thead > tr > th.first.checkbox-column.checkbox > input"))).click()
Error:-
File "D:/Vipul Garg Backup/XXXX/TestingPardotExportProgramsData.py", line 74, in <module>
ProgramsPathlist = getDataForPrograms(RecordsonOnePage,totalnofpages)
File "D:/Vipul Garg Backup/XXXX/TestingPardotExportProgramsData.py", line 63, in getDataForPrograms
RecordsonOnePage = getrecordsoNpage()
File "D:/Vipul Garg Backup/XXXX/TestingPardotExportProgramsData.py", line 42, in getrecordsoNpage
CheckBoxclick = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "#workflow_table > thead > tr > th.first.checkbox-column.checkbox > input"))).click()
File "C:\Users\Vipul.Garg\AppData\Local\Programs\Python\Python37\lib\site-packages\selenium\webdriver\remote\webelement.py", line 80, in click
self._execute(Command.CLICK_ELEMENT)
File "C:\Users\Vipul.Garg\AppData\Local\Programs\Python\Python37\lib\site-packages\selenium\webdriver\remote\webelement.py", line 633, in _execute
return self._parent.execute(command, params)
File "C:\Users\Vipul.Garg\AppData\Local\Programs\Python\Python37\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "C:\Users\Vipul.Garg\AppData\Local\Programs\Python\Python37\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.StaleElementReferenceException: Message: stale element reference: element is not attached to the page document
(Session info: chrome=80.0.3987.122)
But Above Line of Code is Working Intermetantly and throwing error as above.
Any suggestions for this line of code replacement as why I am getting such inconsistency.
You are using ID of the Table and checkbox is inside that table, so you need to pass locator of the check box.
As there is no id defined for the checkbox, so using xpath to reach the locator like - //input[#type='checkbox']
I'm new to python coding. And I'm trying to play around with python and Selenium in chrome. And I have this code where I'm trying to move a webpage up and down. But it isn't working.
body_elem = browser.find_element_by_tag_name('body')
for __ in range(3):
body_elem.send_keys(Keys.END)
sleep(2)
body_elem.send_keys(Keys.HOME)
sleep(2)
All I get is this long error message:
Traceback (most recent call last):
File "selenium_test.py", line 34, in <module>
body_elem.send_keys(Keys.HOME)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/selenium/webdriver/remote/webelement.py", line 479, in send_keys
'value': keys_to_typing(value)})
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/selenium/webdriver/remote/webelement.py", line 628, in _execute
return self._parent.execute(command, params)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/selenium/webdriver/remote/webdriver.py", line 312, in execute
self.error_handler.check_response(response)
File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/selenium/webdriver/remote/errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.StaleElementReferenceException: Message: stale element reference: element is not attached to the page document
Anybody got an idea of whats is wrong?
This is most likely because of the fact that the DOM tree or the body element signature itself changed after the first "down" move. One straightforward way to approach the problem would be to "re-find" the element constantly:
for _ in range(3):
browser.find_element_by_tag_name('body').send_keys(Keys.END)
sleep(2)
browser.find_element_by_tag_name('body').send_keys(Keys.HOME)
sleep(2)
I'm trying to get the html of a page that only loads about 33% of the time. My strategy is to just keep refreshing the page until it eventually loads.
I call this function from another, in which I've already initiated my driver (edited to include a try/catch block for the while statement as per #crookedleaf's suggestion:
def get_table(url, driver):
driver.get(url)
main_window = driver.current_window_handle
html_button = driver.find_element(By.XPATH, '//*[#title="View as HTML"]')
html_button.send_keys(Keys.CONTROL + Keys.RETURN)
driver.find_element_by_tag_name('body').send_keys(Keys.CONTROL + Keys.TAB)
driver.switch_to.active_element
try:
while 'extranet.chem' not in driver.title:
sleep(2)
print('refreshing to get data')
try:
html_button.send_keys(Keys.CONTROL + Keys.RETURN)
except Exception:
print('deeper exception')
driver.refresh()
except:
print('while exception')
pass
I use a nested except to catch possible exceptions to my driver.refresh() call. For some reason, even though I call pass to ignore exceptions, the loop breaks while looking for the driver's title:
the error message:
refreshing to get data
refreshing to get data
refreshing to get data
deeper exception
while exception
Traceback (most recent call last):
File "scraper.py", line 83, in <module>
get_latest()
File "scraper.py", line 28, in get_latest
url = row.find_element(By.XPATH, link_xpath).get_attribute('href')
File "C:\Users\Joseph\AppData\Local\Programs\Python\Python36\lib\site-packages\selenium\webdriver\remote\webelement.py", line 645, in find_element
{"using": by, "value": value})['value']
File "C:\Users\Joseph\AppData\Local\Programs\Python\Python36\lib\site-packages\selenium\webdriver\remote\webelement.py", line 628, in _execute
return self._parent.execute(command, params)
File "C:\Users\Joseph\AppData\Local\Programs\Python\Python36\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 312, in execute
self.error_handler.check_response(response)
File "C:\Users\Joseph\AppData\Local\Programs\Python\Python36\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 237, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.StaleElementReferenceException: Message: The element reference of <tr class="ms-alternating"> is stale; either the element is no longer attached to the DOM, it is not in the current frame context, or the document has been refreshed
Why is this exception not just being ignored?
Looking at the provided traceback, you can see the exception is being raised on the line while 'extranet.chem' not in driver.title::
File "scraper.py", line 55, in get_table
while 'extranet.chem' not in driver.title:
which is not in a try/except block. I'm not sure i've seen that exact exception raised while checking driver.title, but i suppose it is normal. Without knowing anything about the page you are working with, we aren't able to help you much more than that. your option is to catch the exception that is generated at that line. with an alert box being present, you will more than likely be prevented from navigating away from or refreshing that page until the alert is dealt with. you should build in a way to handle the alert.
I am trying to save the page as entire html using selenium, page has a dropdown list in it. There are 2 elements in that list and I want to save the entire html after clicking each of them.
Below is the code I am using:
devlink='http://www.t-mobile.com/content/tmo-wem/en/index/cell-phones/nokia-lumia-521.html'
devname= devlink[devlink.rfind("/")+1:]
ignore_tags=('script','noscript','style')
with contextlib.closing(webdriver.Firefox()) as browser:
browser.get(devlink) # Load page
dropdown = browser.find_element_by_class_name('dropdown-menu')
elements = dropdown.find_elements_by_tag_name('li')
i=0
for element in elements:
element.click()
i = i+1
time.sleep(2)
content=browser.page_source
cleaner=clean.Cleaner()
content=cleaner.clean_html(content)
with open(str(i)+devname,'w') as f:
f.write(content.encode('utf-8'))
This code is giving me a error, traceback of which looks like this Traceback (most recent call last):
File "D:\Windows Phone\Pricing\August\2013-08-05\tmo test\tmo_us_selenium.py", line 52, in <module>
element.click()
File "C:\Python27\lib\site-packages\selenium-2.33.0-py2.7.egg\selenium\webdriver\remote\webelement.py", line 54, in click
self._execute(Command.CLICK_ELEMENT)
File "C:\Python27\lib\site-packages\selenium-2.33.0-py2.7.egg\selenium\webdriver\remote\webelement.py", line 228, in _execute
return self._parent.execute(command, params)
File "C:\Python27\lib\site-packages\selenium-2.33.0-py2.7.egg\selenium\webdriver\remote\webdriver.py", line 165, in execute
self.error_handler.check_response(response)
File "C:\Python27\lib\site-packages\selenium-2.33.0-py2.7.egg\selenium\webdriver\remote\errorhandler.py", line 158, in check_response
raise exception_class(message, screen, stacktrace)
ElementNotVisibleException: Message: u'Element is not currently visible and so may not be interacted with' ; Stacktrace:
at fxdriver.preconditions.visible (file:///c:/users/abhina~1.tal/appdata/local/temp/tmpwnu8em/extensions/fxdriver#googlecode.com/components/command_processor.js:7736)
at DelayedCommand.prototype.checkPreconditions_ (file:///c:/users/abhina~1.tal/appdata/local/temp/tmpwnu8em/extensions/fxdriver#googlecode.com/components/command_processor.js:10437)
at DelayedCommand.prototype.executeInternal_/h (file:///c:/users/abhina~1.tal/appdata/local/temp/tmpwnu8em/extensions/fxdriver#googlecode.com/components/command_processor.js:10456)
at DelayedCommand.prototype.executeInternal_ (file:///c:/users/abhina~1.tal/appdata/local/temp/tmpwnu8em/extensions/fxdriver#googlecode.com/components/command_processor.js:10461)
at DelayedCommand.prototype.execute/< (file:///c:/users/abhina~1.tal/appdata/local/temp/tmpwnu8em/extensions/fxdriver#googlecode.com/components/command_processor.js:10401)
This code is not able to click the 2 elements it is finding, please help me in resolving this issue.
Selenium 2.33 supports Firefox 21 and older. You will have to wait for the Selenium update to have it support FF 22.
Refer to this thread for further details. There are many threads on google that discuss similar issues with Selenium 2.33 and FF 22