I try to click the hidden button by scrolling to the element but still not working. Only if i scroll it manually, the coding is working. May I know the reasons? Thank you!
website: https://www1.hkexnews.hk/search/titlesearch.xhtml?lang=en
options = webdriver.ChromeOptions()
options.add_argument("--no-sandbox")
driver = webdriver.Chrome("C:/Users/Ivan.Chak/Desktop/cbbc/chromedriver", chrome_options=options)
driver.implicitly_wait(30)
driver.get("https://www1.hkexnews.hk/search/titlesearch.xhtml?lang=en")
driver.find_element_by_css_selector("#tier1-select .combobox-input-wrap").click()
driver.find_element_by_css_selector(".searchType .droplist-item:nth-child(2) > a").click()
driver.find_element_by_css_selector("#rbAfter2006 .combobox-field").click()
element = driver.find_element_by_xpath("//a[contains(text(),'Debt and Structured Products')]")
actions = ActionChains(driver)
actions.move_to_element(element).perform()
time.sleep(1)
element.click() <--- This is not working!!!
---------------------------------------------------------------------------
ElementNotInteractableException Traceback (most recent call last)
<ipython-input-323-cf25f85114ab> in <module>()
----> 1 element.click()
C:\ProgramData\Anaconda3\lib\site-packages\selenium\webdriver\remote\webelement.py in click(self)
78 def click(self):
79 """Clicks the element."""
---> 80 self._execute(Command.CLICK_ELEMENT)
81
82 def submit(self):
C:\ProgramData\Anaconda3\lib\site-packages\selenium\webdriver\remote\webelement.py in _execute(self, command, params)
631 params = {}
632 params['id'] = self._id
--> 633 return self._parent.execute(command, params)
634
635 def find_element(self, by=By.ID, value=None):
C:\ProgramData\Anaconda3\lib\site-packages\selenium\webdriver\remote\webdriver.py in execute(self, driver_command, params)
319 response = self.command_executor.execute(driver_command, params)
320 if response:
--> 321 self.error_handler.check_response(response)
322 response['value'] = self._unwrap_value(
323 response.get('value', None))
C:\ProgramData\Anaconda3\lib\site-packages\selenium\webdriver\remote\errorhandler.py in check_response(self, response)
240 alert_text = value['alert'].get('text')
241 raise exception_class(message, screen, stacktrace, alert_text)
--> 242 raise exception_class(message, screen, stacktrace)
243
244 def _value_or_default(self, obj, key, default):
ElementNotInteractableException: Message: element not interactable
(Session info: chrome=75.0.3770.142)
It might be possible you are scrolling to the element(button) . Which again is getting hidden if your webpage have any kind of ribbon. Try scrolling to any other element above it. This had solved my problem.
You have to scroll to the each menu item before click, for that you can use JavaScript arguments[0].scrollIntoView(true);
You can use general method to scroll before click like in code below:
def click(element):
driver.execute_script("arguments[0].scrollIntoView(true);", element)
element.click()
driver = webdriver.Chrome()
driver.get("https://www1.hkexnews.hk/search/titlesearch.xhtml?lang=en")
driver.find_element_by_css_selector(".searchType").click()
driver.find_element_by_xpath("//div[#class='droplist-item' and ./a[.='Headline Category']]").click()
driver.find_element_by_id("rbAfter2006").click()
click(driver.find_element_by_xpath("//a[.='Debt and Structured Products']"))
click(driver.find_element_by_xpath("//a[.='Debt Securities']"))
click(driver.find_element_by_xpath("//a[.='Issuer-Specific Report - Debt Securities']"))
Or you can use specific method for menu selection like below:
def select_menu(menu_text):
element = driver.find_element_by_xpath(f"//a[.='{menu_text}']")
driver.execute_script("arguments[0].scrollIntoView(true);", element)
element.click()
driver = webdriver.Chrome()
driver.get("https://www1.hkexnews.hk/search/titlesearch.xhtml?lang=en")
driver.find_element_by_css_selector(".searchType").click()
driver.find_element_by_xpath("//div[#class='droplist-item' and ./a[.='Headline Category']]").click()
driver.find_element_by_id("rbAfter2006").click()
select_menu("Debt and Structured Products")
select_menu("Debt Securities")
select_menu("Issuer-Specific Report - Debt Securities")
More generic method to select search criterias:
def select_search_criteria(criteria, *menus):
driver.find_element_by_css_selector(".searchType").click()
criteria_element = driver.find_element_by_xpath(f"//div[#class='droplist-item' and ./a[.='{criteria}']]")
data_value = criteria_element.get_attribute("data-value")
criteria_element.click()
document_type_element = driver.find_element_by_id(data_value)
if document_type_element.find_element_by_css_selector("a.combobox-field").get_attribute("aria-expanded") == "false":
document_type_element.click()
for menu in menus:
element = document_type_element.find_element_by_xpath(f".//a[.='{menu}']")
driver.execute_script("arguments[0].scrollIntoView(true);", element)
element.click()
driver = webdriver.Chrome()
driver.get("https://www1.hkexnews.hk/search/titlesearch.xhtml?lang=en")
select_search_type("Headline Category", "Debt and Structured Products", "Debt Securities")
select_search_type("Document Type", "Circulars")
Related
I have a page with source code like the code below. In it after I take an action an “Undo” and a “Close” button appear. I’m trying to click the “Close” button. I’ve tried all three of the chunks of code below, none are working. Can someone please point out what I’m doing wrong, or suggest something else to try?
html source:
<div class="_3Aslx7L3GVI4XM7PUyYKza action-bar"><div class="container"><i class="success-icon fontello-ok-circle"></i><div class="success-message">Your stuff is going to <span>place</span> is on its way.</div><div class="gh69ID1m3_xtdTUQuwadU"><button class="c-button c-button--gray"> Undo</button></div><div class="gh69ID1m3_xtdTUQuwadU"><button class="c-button c-button--blue"> Close</button></div></div></div>
code attempts:
#driver.find_element_by_id("gh69ID1m3_xtdTUQuwadU").click()
driver.find_element_by_css_selector('.c-button.c-button--blue').click()
#driver.find_element_by_link_text('Close').click()
error:
---------------------------------------------------------------------------
ElementNotVisibleException Traceback (most recent call last)
<ipython-input-15-6d570be770d7> in <module>()
1 #driver.find_element_by_id("gh69ID1m3_xtdTUQuwadU").click()
----> 2 driver.find_element_by_css_selector('.c-button.c-button--blue').click()
3 #driver.find_element_by_link_text('Close').click()
~/anaconda/envs/py36/lib/python3.6/site-packages/selenium/webdriver/remote/webelement.py in click(self)
78 def click(self):
79 """Clicks the element."""
---> 80 self._execute(Command.CLICK_ELEMENT)
81
82 def submit(self):
~/anaconda/envs/py36/lib/python3.6/site-packages/selenium/webdriver/remote/webelement.py in _execute(self, command, params)
626 params = {}
627 params['id'] = self._id
--> 628 return self._parent.execute(command, params)
629
630 def find_element(self, by=By.ID, value=None):
~/anaconda/envs/py36/lib/python3.6/site-packages/selenium/webdriver/remote/webdriver.py in execute(self, driver_command, params)
318 response = self.command_executor.execute(driver_command, params)
319 if response:
--> 320 self.error_handler.check_response(response)
321 response['value'] = self._unwrap_value(
322 response.get('value', None))
~/anaconda/envs/py36/lib/python3.6/site-packages/selenium/webdriver/remote/errorhandler.py in check_response(self, response)
240 alert_text = value['alert'].get('text')
241 raise exception_class(message, screen, stacktrace, alert_text)
--> 242 raise exception_class(message, screen, stacktrace)
243
244 def _value_or_default(self, obj, key, default):
ElementNotVisibleException: Message: element not interactable
(Session info: chrome=72.0.3626.109)
(Driver info: chromedriver=2.42.591059 (a3d9684d10d61aa0c45f6723b327283be1ebaad8),platform=Mac OS X 10.12.6 x86_64)
The element with text as Close is a dynamic element so to locate the element you have to induce WebDriverWait for the element to be clickable and you can use either of the following solutions:
Using CSS_SELECTOR:
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div.action-bar button.c-button.c-button--blue"))).click()
Using XPATH:
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//div[contains(#class, 'action-bar')]//button[#class='c-button c-button--blue' and normalize-space()='Close']"))).click()
Note : You have to add the following imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
Try using XPATHs and Action class for this kind of scenarios.
BTN_xpath = //*[contains(#class, 'c-button--blue')]
WebElement btn = driver.find_element_by_xpath(BTN_xpath);
Actions ac = new Actions(driver);
ac.click(btn).build().perform();
Also, if you are using the Firefox browser, there could be issues where its not able to interact with elements that were not visible before in the page. So, try checking a with a different browser as well.
I'm writing a scraper using Selenium, and it was working just fine until 15 minutes ago..
All of sudden, I get the following error every time I execute. This is my code:
searchDate = wait.until(EC.element_to_be_clickable((By.XPATH, "/input[#placeholder='select sold date range']")))
searchDate.click()
time.sleep(5)
And the error:
WebDriverException: Message: chrome not reachable
(Session info: chrome=80.0.3987.163)
And my chrome setting:
options = webdriver.ChromeOptions()
prefs = {'download.default_directory': new_dir}
options.add_experimental_option('prefs', prefs)
browser = webdriver.Chrome(executable_path = r'C:/Program Files/chromedriver/chromedriver.exe')
I tried to solve the problem by chain the options as other post suggested, such as:
options = webdriver.ChromeOptions()
options.add_argument('--no-sandbox')
prefs = {'download.default_directory': new_dir}
options.add_experimental_option('prefs', prefs)
browser = webdriver.Chrome(executable_path = r'C:/Program Files/chromedriver/chromedriver.exe', chrome_options=options)
But I still get the exact same error. Why did it happen?
Any advice will be greatly appreciated!
++
---------------------------------------------------------------------------
WebDriverException Traceback (most recent call last)
<ipython-input-102-c7d533e1df88> in <module>
----> 1 wait.until(EC.visibility_of_element_located((By.XPATH,"//input[#name='daterangepicker_start']"))).click()
2
3
~\AppData\Roaming\Python\Python37\site-packages\selenium\webdriver\support\wait.py in until(self, method, message)
69 while True:
70 try:
---> 71 value = method(self._driver)
72 if value:
73 return value
~\AppData\Roaming\Python\Python37\site-packages\selenium\webdriver\support\expected_conditions.py in __call__(self, driver)
126 def __call__(self, driver):
127 try:
--> 128 return _element_if_visible(_find_element(driver, self.locator))
129 except StaleElementReferenceException:
130 return False
~\AppData\Roaming\Python\Python37\site-packages\selenium\webdriver\support\expected_conditions.py in _find_element(driver, by)
413 raise e
414 except WebDriverException as e:
--> 415 raise e
416
417
~\AppData\Roaming\Python\Python37\site-packages\selenium\webdriver\support\expected_conditions.py in _find_element(driver, by)
409 if thrown."""
410 try:
--> 411 return driver.find_element(*by)
412 except NoSuchElementException as e:
413 raise e
~\AppData\Roaming\Python\Python37\site-packages\selenium\webdriver\remote\webdriver.py in find_element(self, by, value)
976 return self.execute(Command.FIND_ELEMENT, {
977 'using': by,
--> 978 'value': value})['value']
979
980 def find_elements(self, by=By.ID, value=None):
~\AppData\Roaming\Python\Python37\site-packages\selenium\webdriver\remote\webdriver.py in execute(self, driver_command, params)
319 response = self.command_executor.execute(driver_command, params)
320 if response:
--> 321 self.error_handler.check_response(response)
322 response['value'] = self._unwrap_value(
323 response.get('value', None))
~\AppData\Roaming\Python\Python37\site-packages\selenium\webdriver\remote\errorhandler.py in check_response(self, response)
240 alert_text = value['alert'].get('text')
241 raise exception_class(message, screen, stacktrace, alert_text)
--> 242 raise exception_class(message, screen, stacktrace)
243
244 def _value_or_default(self, obj, key, default):
WebDriverException: Message: chrome not reachable
(Session info: chrome=81.0.4044.92)
Can you please check if your chrome browser is compatible with your chrome browser?
Could be possible your chrome is autommatically updated and now your chrome browser is not compatible with chrome driver.
You can download chrome driver from here
I use selenium webdriver to do a search on a website which gives me multiple pages with several links on each page. My goal is to create a list of all links the search produced.
It works fine for the first page but when I'm on the second page the for loop where I try to extract the 'href' attribute produces a StaleElementReferenceException.
I believe there should be some way to fix this with WebDriverWait but I can't quite figure out how exactly. Thanks for any suggestions.
links =[]
while True:
result = driver.find_element_by_id('results')
# list of all relevant elements on the page
docs = result.find_elements_by_xpath('//li[#class="row1"]')
for d in docs:
# this line produces StaleElementReferenceException
link = d.find_element_by_tag_name('a')
links.append(link.get_attribute('href'))
# try block checks if "next page" button exists and clicks it (this works fine)
try:
next_page = WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.XPATH, '//a[contains(#class,"la-TriangleRight")]')))
driver.execute_script('arguments[0].click();', next_page)
# if "next page" button doesn't exist we break out of the loop
except:
break
EDIT:
Error message:
StaleElementReferenceException Traceback (most recent call last)
<ipython-input-6-ad6166a696e6> in <module>
23 # link = WebDriverWait(d, 30).until(EC.presence_of_element_located((By.XPATH, '//a')))
24 link = d.find_element_by_tag_name('a')
---> 25 links.append(link.get_attribute('href'))
26
27 # for i in range(25):
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/selenium/webdriver/remote/webelement.py in get_attribute(self, name)
139 attributeValue = self.parent.execute_script(
140 "return (%s).apply(null, arguments);" % getAttribute_js,
--> 141 self, name)
142 else:
143 resp = self._execute(Command.GET_ELEMENT_ATTRIBUTE, {'name': name})
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/selenium/webdriver/remote/webdriver.py in execute_script(self, script, *args)
634 return self.execute(command, {
635 'script': script,
--> 636 'args': converted_args})['value']
637
638 def execute_async_script(self, script, *args):
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/selenium/webdriver/remote/webdriver.py in execute(self, driver_command, params)
319 response = self.command_executor.execute(driver_command, params)
320 if response:
--> 321 self.error_handler.check_response(response)
322 response['value'] = self._unwrap_value(
323 response.get('value', None))
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/selenium/webdriver/remote/errorhandler.py in check_response(self, response)
240 alert_text = value['alert'].get('text')
241 raise exception_class(message, screen, stacktrace, alert_text)
--> 242 raise exception_class(message, screen, stacktrace)
243
244 def _value_or_default(self, obj, key, default):
StaleElementReferenceException: Message: stale element reference: element is not attached to the page document
(Session info: chrome=75.0.3770.100)
Instead for each loop, please try with for loop with range.
links =[]
while True:
result = driver.find_element_by_id('results')
# list of all relevant elements on the page
docs = result.find_elements_by_xpath('//li[#class="row1"]')
for i in range(len(docs)):
# this line produces StaleElementReferenceException
WebDriverWait(driver, 30).until(EC.staleness_of(docs[i]))
link = docs[i].find_element_by_tag_name('a')
links.append(link.get_attribute('href'))
result = driver.find_element_by_id('results')
docs = result.find_elements_by_xpath('//li[#class="row1"]')
# try block checks if "next page" button exists and clicks it (this works fine)
try:
next_page = WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.XPATH, '//a[contains(#class,"la-TriangleRight")]')))
driver.execute_script('arguments[0].click();', next_page)
# if "next page" button doesn't exist we break out of the loop
except:
break
I have a page with source code like the code below. In it after I take an action an “Undo” and a “Close” button appear. I’m trying to click the “Close” button. I’ve tried all three of the chunks of code below, none are working. Can someone please point out what I’m doing wrong, or suggest something else to try?
html source:
<div class="_3Aslx7L3GVI4XM7PUyYKza action-bar"><div class="container"><i class="success-icon fontello-ok-circle"></i><div class="success-message">Your stuff is going to <span>place</span> is on its way.</div><div class="gh69ID1m3_xtdTUQuwadU"><button class="c-button c-button--gray"> Undo</button></div><div class="gh69ID1m3_xtdTUQuwadU"><button class="c-button c-button--blue"> Close</button></div></div></div>
code attempts:
#driver.find_element_by_id("gh69ID1m3_xtdTUQuwadU").click()
driver.find_element_by_css_selector('.c-button.c-button--blue').click()
#driver.find_element_by_link_text('Close').click()
error:
---------------------------------------------------------------------------
ElementNotVisibleException Traceback (most recent call last)
<ipython-input-15-6d570be770d7> in <module>()
1 #driver.find_element_by_id("gh69ID1m3_xtdTUQuwadU").click()
----> 2 driver.find_element_by_css_selector('.c-button.c-button--blue').click()
3 #driver.find_element_by_link_text('Close').click()
~/anaconda/envs/py36/lib/python3.6/site-packages/selenium/webdriver/remote/webelement.py in click(self)
78 def click(self):
79 """Clicks the element."""
---> 80 self._execute(Command.CLICK_ELEMENT)
81
82 def submit(self):
~/anaconda/envs/py36/lib/python3.6/site-packages/selenium/webdriver/remote/webelement.py in _execute(self, command, params)
626 params = {}
627 params['id'] = self._id
--> 628 return self._parent.execute(command, params)
629
630 def find_element(self, by=By.ID, value=None):
~/anaconda/envs/py36/lib/python3.6/site-packages/selenium/webdriver/remote/webdriver.py in execute(self, driver_command, params)
318 response = self.command_executor.execute(driver_command, params)
319 if response:
--> 320 self.error_handler.check_response(response)
321 response['value'] = self._unwrap_value(
322 response.get('value', None))
~/anaconda/envs/py36/lib/python3.6/site-packages/selenium/webdriver/remote/errorhandler.py in check_response(self, response)
240 alert_text = value['alert'].get('text')
241 raise exception_class(message, screen, stacktrace, alert_text)
--> 242 raise exception_class(message, screen, stacktrace)
243
244 def _value_or_default(self, obj, key, default):
ElementNotVisibleException: Message: element not interactable
(Session info: chrome=72.0.3626.109)
(Driver info: chromedriver=2.42.591059 (a3d9684d10d61aa0c45f6723b327283be1ebaad8),platform=Mac OS X 10.12.6 x86_64)
The element with text as Close is a dynamic element so to locate the element you have to induce WebDriverWait for the element to be clickable and you can use either of the following solutions:
Using CSS_SELECTOR:
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "div.action-bar button.c-button.c-button--blue"))).click()
Using XPATH:
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//div[contains(#class, 'action-bar')]//button[#class='c-button c-button--blue' and normalize-space()='Close']"))).click()
Note : You have to add the following imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
Try using XPATHs and Action class for this kind of scenarios.
BTN_xpath = //*[contains(#class, 'c-button--blue')]
WebElement btn = driver.find_element_by_xpath(BTN_xpath);
Actions ac = new Actions(driver);
ac.click(btn).build().perform();
Also, if you are using the Firefox browser, there could be issues where its not able to interact with elements that were not visible before in the page. So, try checking a with a different browser as well.
please, I'm trying to get information from a public website of the Brazilian congress: name of a voting session, date and table with list of votes
This site: http://www2.camara.leg.br/atividade-legislativa/plenario/chamadaExterna.html?link=http://www.camara.gov.br/internet/votacao/mostraVotacao.asp?ideVotacao=6706&tipo=partido
I used Python 3, selenium webdriver and PhantomJS:
from selenium import webdriver
path_to_phantomjs = '/Users/George/Documents/phantomjs/phantomjs-2.1.1-windows/bin/phantomjs'
browser = webdriver.PhantomJS(executable_path = path_to_phantomjs)
browser.get("http://www2.camara.leg.br/atividade-legislativa/plenario/chamadaExterna.html?link=http://www.camara.gov.br/internet/votacao/mostraVotacao.asp?ideVotacao=6706&tipo=partido")
nome_votacao = browser.find_element_by_xpath("//[#id='corpoVotacao']/p[3]/text()")
data_votacao = browser.find_element_by_xpath("//[#id='corpoVotacao']/div[1]/div[1]/div/div/p[1]/text()[1]")
list_deputados = browser.find_elements_by_xpath(".//table[#class='tabela-2']")
But it looks like I'm wrongly selecting locations
The nome_votacao appears this error message:
---------------------------------------------------------------------------
InvalidSelectorException Traceback (most recent call last)
<ipython-input-11-e67933637ae0> in <module>()
----> 1 nome_votacao = browser.find_element_by_xpath("//[#id='corpoVotacao']/p[3]/text()")
c:\users\george\appdata\local\programs\python\python36-32\code\votos\lib\site-packages\selenium\webdriver\remote\webdriver.py in find_element_by_xpath(self, xpath)
363 driver.find_element_by_xpath('//div/td[1]')
364 """
--> 365 return self.find_element(by=By.XPATH, value=xpath)
366
367 def find_elements_by_xpath(self, xpath):
c:\users\george\appdata\local\programs\python\python36-32\code\votos\lib\site-packages\selenium\webdriver\remote\webdriver.py in find_element(self, by, value)
841 return self.execute(Command.FIND_ELEMENT, {
842 'using': by,
--> 843 'value': value})['value']
844
845 def find_elements(self, by=By.ID, value=None):
c:\users\george\appdata\local\programs\python\python36-32\code\votos\lib\site-packages\selenium\webdriver\remote\webdriver.py in execute(self, driver_command, params)
306 response = self.command_executor.execute(driver_command, params)
307 if response:
--> 308 self.error_handler.check_response(response)
309 response['value'] = self._unwrap_value(
310 response.get('value', None))
c:\users\george\appdata\local\programs\python\python36-32\code\votos\lib\site-packages\selenium\webdriver\remote\errorhandler.py in check_response(self, response)
192 elif exception_class == UnexpectedAlertPresentException and 'alert' in value:
193 raise exception_class(message, screen, stacktrace, value['alert'].get('text'))
--> 194 raise exception_class(message, screen, stacktrace)
195
196 def _value_or_default(self, obj, key, default):
InvalidSelectorException: Message: {"errorMessage":"Unable to locate an element with the xpath expression //[#id='corpoVotacao']/p[3]/text() because of the following error:\nError: INVALID_EXPRESSION_ERR: DOM XPath Exception 51","request":{"headers":{"Accept":"application/json","Accept-Encoding":"identity","Connection":"close","Content-Length":"118","Content-Type":"application/json;charset=UTF-8","Host":"127.0.0.1:55799","User-Agent":"Python http auth"},"httpVersion":"1.1","method":"POST","post":"{\"using\": \"xpath\", \"value\": \"//[#id='corpoVotacao']/p[3]/text()\", \"sessionId\": \"366665f0-be24-11e7-aa25-75da268b98e2\"}","url":"/element","urlParsed":{"anchor":"","query":"","file":"element","directory":"/","path":"/element","relative":"/element","port":"","host":"","password":"","user":"","userInfo":"","authority":"","protocol":"","source":"/element","queryKey":{},"chunks":["element"]},"urlOriginal":"/session/366665f0-be24-11e7-aa25-75da268b98e2/element"}}
Screenshot: available via screen
The data_votacao this error message:
---------------------------------------------------------------------------
InvalidSelectorException Traceback (most recent call last)
<ipython-input-12-24c43341f310> in <module>()
----> 1 data_votacao = browser.find_element_by_xpath("//[#id='corpoVotacao']/div[1]/div[1]/div/div/p[1]/text()[1]")
c:\users\george\appdata\local\programs\python\python36-32\code\votos\lib\site-packages\selenium\webdriver\remote\webdriver.py in find_element_by_xpath(self, xpath)
363 driver.find_element_by_xpath('//div/td[1]')
364 """
--> 365 return self.find_element(by=By.XPATH, value=xpath)
366
367 def find_elements_by_xpath(self, xpath):
c:\users\george\appdata\local\programs\python\python36-32\code\votos\lib\site-packages\selenium\webdriver\remote\webdriver.py in find_element(self, by, value)
841 return self.execute(Command.FIND_ELEMENT, {
842 'using': by,
--> 843 'value': value})['value']
844
845 def find_elements(self, by=By.ID, value=None):
c:\users\george\appdata\local\programs\python\python36-32\code\votos\lib\site-packages\selenium\webdriver\remote\webdriver.py in execute(self, driver_command, params)
306 response = self.command_executor.execute(driver_command, params)
307 if response:
--> 308 self.error_handler.check_response(response)
309 response['value'] = self._unwrap_value(
310 response.get('value', None))
c:\users\george\appdata\local\programs\python\python36-32\code\votos\lib\site-packages\selenium\webdriver\remote\errorhandler.py in check_response(self, response)
192 elif exception_class == UnexpectedAlertPresentException and 'alert' in value:
193 raise exception_class(message, screen, stacktrace, value['alert'].get('text'))
--> 194 raise exception_class(message, screen, stacktrace)
195
196 def _value_or_default(self, obj, key, default):
InvalidSelectorException: Message: {"errorMessage":"Unable to locate an element with the xpath expression //[#id='corpoVotacao']/div[1]/div[1]/div/div/p[1]/text()[1] because of the following error:\nError: INVALID_EXPRESSION_ERR: DOM XPath Exception 51","request":{"headers":{"Accept":"application/json","Accept-Encoding":"identity","Connection":"close","Content-Length":"143","Content-Type":"application/json;charset=UTF-8","Host":"127.0.0.1:55799","User-Agent":"Python http auth"},"httpVersion":"1.1","method":"POST","post":"{\"using\": \"xpath\", \"value\": \"//[#id='corpoVotacao']/div[1]/div[1]/div/div/p[1]/text()[1]\", \"sessionId\": \"366665f0-be24-11e7-aa25-75da268b98e2\"}","url":"/element","urlParsed":{"anchor":"","query":"","file":"element","directory":"/","path":"/element","relative":"/element","port":"","host":"","password":"","user":"","userInfo":"","authority":"","protocol":"","source":"/element","queryKey":{},"chunks":["element"]},"urlOriginal":"/session/366665f0-be24-11e7-aa25-75da268b98e2/element"}}
Screenshot: available via screen
And the list_deputados generates an empty list
The data_votacao was to have this content: "10/11/2015 20:15". And it has this XPath in the inspect: //*[#id="corpoVotacao"]/p[2]/text()[1]
The nome_votação was to have this content: "MPV Nº 688/2015 - PROJETO DE LEI DE CONVERSÃO - Nominal Eletrônica". And it has this XPath in the inspect: //*[#id="corpoVotacao"]/p[3]/text()
And the list_deputados was to have the complete table with the votes (names, UF and vote), starting in the line "Parlamentar". It has this XPath: //*[#id="listagem"]/table. And the class="tabela-2"
Does anyone know the correct form of the commands? Or some tutorial?
The first problem is that all the interested elements are inside an iframe.
So, first you have to switch to the iframe:
iframe = driver.find_element_by_xpath("//iframe[#id='ifr']")
driver.switch_to_frame(iframe)
Also the xpath aren't correct. You could use the following:
nome_votacao = driver.find_element_by_xpath("//*[#id='corpoVotacao']/p[3]")
data_votacao = driver.find_element_by_xpath("//*[#id='corpoVotacao']/div[1]/div[1]/div/div/p[1]")
list_deputados = driver.find_elements_by_xpath("*//div[#id='listagem']/table[#class='tabela-2']/tbody/tr/td")
The entire code:
driver = webdriver.PhantomJS(executable_path=r'/pathTo/phantomjs')
driver.get("http://www2.camara.leg.br/atividade-legislativa/plenario/chamadaExterna.html?link=http://www.camara.gov.br/internet/votacao/mostraVotacao.asp?ideVotacao=6706&tipo=partido")
iframe = driver.find_element_by_xpath("//iframe[#id='ifr']")
driver.switch_to_frame(iframe)
nome_votacao = driver.find_element_by_xpath("//*[#id='corpoVotacao']/p[3]")
data_votacao = driver.find_element_by_xpath("//*[#id='corpoVotacao']/div[1]/div[1]/div/div/p[1]")
list_deputados = driver.find_elements_by_xpath("*//div[#id='listagem']/table[#class='tabela-2']/tbody/tr/td")
print(nome_votacao.text)
print("----------------------------")
print(data_votacao.text)
print("----------------------------")
i=0
maxLen=len(list_deputados)
print("maxLen: " + str(maxLen))
for deputado in list_deputados:
print("Parlamentar DEM: " + list_deputados[i].text)
print("UF: " + list_deputados[i+1].text)
print("Voto: " + list_deputados[i+2].text)
i=i+2
print("--------------"+str(i)+"-------------")
if(i==maxLen-3): #don't consider the last row: Total Solidaried: 11
break
driver.close()
Can you try
//*[#id="corpoVotacao"]/p[3]/text() as XPATH instead of
//[#id='corpoVotacao']/p[3]/text()
//*[#id="corpoVotacao"]/div[1]/div[1]/div/div/p[1]/text()[1] as XPATH instead of
//[#id='corpoVotacao']/div[1]/div[1]/div/div/p[1]/text()[1]
last XPATH seems correct to me, also can you have implicit wait after get() just to ensure elements are loaded on UI.