Selenium problem in VS Code Interactive Mode - python

I am trying to run Selenium from the interactive mode of Visual Studio code, but I am getting the error below. Everything works great in Jupyter or when I run the script non-interactively.
MRE
# %%
from selenium import webdriver
url = 'google.com'
driver = webdriver.Firefox()
driver.get(url)
Error message
---------------------------------------------------------------------------
WebDriverException Traceback (most recent call last)
++redacted++.py in
3 url = 'google.com'
----> 4 driver = webdriver.Firefox()
5 driver.get(url)
~/anaconda3/lib/python3.8/site-packages/selenium/webdriver/firefox/webdriver.py in __init__(self, firefox_profile, firefox_binary, timeout, capabilities, proxy, executable_path, options, service_log_path, firefox_options, service_args, desired_capabilities, log_path, keep_alive)
168 executor = FirefoxRemoteConnection(
169 remote_server_addr=self.service.service_url)
--> 170 RemoteWebDriver.__init__(
171 self,
172 command_executor=executor,
~/anaconda3/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py in __init__(self, command_executor, desired_capabilities, browser_profile, proxy, keep_alive, file_detector, options)
155 warnings.warn("Please use FirefoxOptions to set browser profile",
156 DeprecationWarning, stacklevel=2)
--> 157 self.start_session(capabilities, browser_profile)
158 self._switch_to = SwitchTo(self)
159 self._mobile = Mobile(self)
~/anaconda3/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py in start_session(self, capabilities, browser_profile)
250 parameters = {"capabilities": w3c_caps,
251 "desiredCapabilities": capabilities}
--> 252 response = self.execute(Command.NEW_SESSION, parameters)
253 if 'sessionId' not in response:
254 response = response['value']
~/anaconda3/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py in execute(self, driver_command, params)
319 response = self.command_executor.execute(driver_command, params)
320 if response:
--> 321 self.error_handler.check_response(response)
322 response['value'] = self._unwrap_value(
323 response.get('value', None))
~/anaconda3/lib/python3.8/site-packages/selenium/webdriver/remote/errorhandler.py in check_response(self, response)
240 alert_text = value['alert'].get('text')
241 raise exception_class(message, screen, stacktrace, alert_text)
--> 242 raise exception_class(message, screen, stacktrace)
243
244 def _value_or_default(self, obj, key, default):
WebDriverException: Message: Process unexpectedly closed with status 11
Edit 1:
Adding a screenshot of available environments
The first complain about ipykernel not being installed, even though it should be according to pip. The last two give the error

Related

Selenium webdriver -> closing my chrome after one second

I have issue with selenium
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
options = Options()
options.binary_location = "C:\\Program Files\\Google\\Chrome Beta\\Application\\chrome.exe"
driver = webdriver.Chrome(options = options, executable_path=r'C:\\Users\\Mariusz\\anaconda3\\envs\\pgg\\Lib\\site-packages\\chromedriver_binary\\chromedriver.exe')
driver.get('http://google.com/')
print("Chrome Browser Invoked successfully")
driver.quit()
import chromedriver_binary
wd = wd.Chrome()
wd.implicitly_wait(10)
Above code starting my Chrome for 1 second and then immediately close.
I have feedback in Anaconda:
Chrome Browser Invoked successfully
And debug:
WebDriverException Traceback (most recent call
last) Input In [25], in <cell line: 12>()
9 driver.quit()
10 import chromedriver_binary
---> 12 wd = wd.Chrome()
13 wd.implicitly_wait(10)
File
~\anaconda3\envs\pgg\lib\site-packages\selenium\webdriver\chrome\webdriver.py:76,
in WebDriver.__init__(self, executable_path, port, options,
service_args, desired_capabilities, service_log_path, chrome_options,
keep_alive)
73 self.service.start()
75 try:
---> 76 RemoteWebDriver.__init__(
77 self,
78 command_executor=ChromeRemoteConnection(
79 remote_server_addr=self.service.service_url,
80 keep_alive=keep_alive),
81 desired_capabilities=desired_capabilities)
82 except Exception:
83 self.quit()
File
~\anaconda3\envs\pgg\lib\site-packages\selenium\webdriver\remote\webdriver.py:157,
in WebDriver.__init__(self, command_executor, desired_capabilities,
browser_profile, proxy, keep_alive, file_detector, options)
154 if browser_profile is not None:
155 warnings.warn("Please use FirefoxOptions to set browser profile",
156 DeprecationWarning, stacklevel=2)
--> 157 self.start_session(capabilities, browser_profile)
158 self._switch_to = SwitchTo(self)
159 self._mobile = Mobile(self)
File
~\anaconda3\envs\pgg\lib\site-packages\selenium\webdriver\remote\webdriver.py:252,
in WebDriver.start_session(self, capabilities, browser_profile)
249 w3c_caps = _make_w3c_caps(capabilities)
250 parameters = {"capabilities": w3c_caps,
251 "desiredCapabilities": capabilities}
--> 252 response = self.execute(Command.NEW_SESSION, parameters)
253 if 'sessionId' not in response:
254 response = response['value']
File
~\anaconda3\envs\pgg\lib\site-packages\selenium\webdriver\remote\webdriver.py:321,
in WebDriver.execute(self, driver_command, params)
319 response = self.command_executor.execute(driver_command, params)
320 if response:
--> 321 self.error_handler.check_response(response)
322 response['value'] = self._unwrap_value(
323 response.get('value', None))
324 return response
File
~\anaconda3\envs\pgg\lib\site-packages\selenium\webdriver\remote\errorhandler.py:242,
in ErrorHandler.check_response(self, response)
240 alert_text = value['alert'].get('text')
241 raise exception_class(message, screen, stacktrace, alert_text)
--> 242 raise exception_class(message, screen, stacktrace)
WebDriverException: Message: unknown error: cannot find Chrome binary
I think that I've declared chrome binary there:
options.binary_location = "C:\\Program Files\\Google\\Chrome Beta\\Application\\chrome.exe"
Ok I fixed this by remove last lines:
driver.quit()
import chromedriver_binary
wd = wd.Chrome()
wd.implicitly_wait(10)

Altair saver - frequent errors saving charts

I've been getting frequent but not constant errors using Altair saver to save .png files via selenium and chromedriver. First I get a "headless chrome" error:
WebDriverException: Message: unknown error: session deleted because of
page crash from unknown error: cannot determine loading status from
tab crashed (Session info: headless chrome=90.0.4430.212)
Followed by a "invalid session" error when I try to save it again:
InvalidSessionIdException: Message: invalid session id
Based on my read of StackOverflow and elsewhere, the initial error leaves a chromedriver session open and that causes the second error. It will resolve itself if I close out of everything and reopen but that's not really a workable solution. Any advice on making saving Altair charts more reliable?
Versions are up to date and compatible as far as I know: altair 4.1.0, altair-saver 0.5.0, selenium 3.141.0, Chrome 90.0.4430.212, ChromeDriver 90.0.4430.24, chromedriver-binary 90.0.4430.24.0
Full trackback of headless chrome error:
---------------------------------------------------------------------------
WebDriverException Traceback (most recent call last)
<ipython-input-37-cf20dd94d732> in <module>
---> 30 chart.save('chart.png', scale_factor=3)
c:\programdata\anaconda3\envs\[...]\lib\site-packages\altair\vegalite\v4\api.py in save(self, fp, format, override_data_transformer, scale_factor, vegalite_version, vega_version, vegaembed_version, **kwargs)
474 if override_data_transformer:
475 with data_transformers.disable_max_rows():
--> 476 result = save(**kwds)
477 else:
478 result = save(**kwds)
c:\programdata\anaconda3\envs\[...]\lib\site-packages\altair\utils\save.py in save(chart, fp, vega_version, vegaembed_version, format, mode, vegalite_version, embed_options, json_kwds, webdriver, scale_factor, **kwargs)
119 webdriver=webdriver,
120 scale_factor=scale_factor,
--> 121 **kwargs,
122 )
123 if format == "png":
c:\programdata\anaconda3\envs\[...]\lib\site-packages\altair\utils\mimebundle.py in spec_to_mimebundle(spec, format, mode, vega_version, vegaembed_version, vegalite_version, **kwargs)
58 "see http://github.com/altair-viz/altair_saver/".format(fmt=format)
59 )
---> 60 return altair_saver.render(spec, format, mode=mode, **kwargs)
61 if format == "html":
62 html = spec_to_html(
c:\programdata\anaconda3\envs\[...]\lib\site-packages\altair_saver\_core.py in render(chart, fmts, mode, embed_options, method, **kwargs)
255 Saver = _select_saver(method, mode=mode, fmt=fmt)
256 saver = Saver(spec, mode=mode, embed_options=embed_options, **kwargs)
--> 257 mimebundle.update(saver.mimebundle(fmt))
258
259 return mimebundle
c:\programdata\anaconda3\envs\[...]\lib\site-packages\altair_saver\savers\_saver.py in mimebundle(self, fmts)
88 vegalite_version=self._package_versions["vega-lite"],
89 )
---> 90 bundle[mimetype] = self._serialize(fmt, "mimebundle")
91 return bundle
92
c:\programdata\anaconda3\envs\[...]\lib\site-packages\altair_saver\savers\_selenium.py in _serialize(self, fmt, content_type)
282
283 def _serialize(self, fmt: str, content_type: str) -> MimebundleContent:
--> 284 out = self._extract(fmt)
285 if fmt == "png":
286 assert isinstance(out, str)
c:\programdata\anaconda3\envs\[...]\lib\site-packages\altair_saver\savers\_selenium.py in _extract(self, fmt)
262
263 url = self._serve(html, js_resources)
--> 264 driver.get("about:blank")
265 driver.get(url)
266 try:
c:\programdata\anaconda3\envs\[...]\lib\site-packages\selenium\webdriver\remote\webdriver.py in get(self, url)
331 Loads a web page in the current browser session.
332 """
--> 333 self.execute(Command.GET, {'url': url})
334
335 #property
c:\programdata\anaconda3\envs\[...]\lib\site-packages\selenium\webdriver\remote\webdriver.py in execute(self, driver_command, params)
319 response = self.command_executor.execute(driver_command, params)
320 if response:
--> 321 self.error_handler.check_response(response)
322 response['value'] = self._unwrap_value(
323 response.get('value', None))
c:\programdata\anaconda3\envs\[...]\lib\site-packages\selenium\webdriver\remote\errorhandler.py in check_response(self, response)
240 alert_text = value['alert'].get('text')
241 raise exception_class(message, screen, stacktrace, alert_text)
--> 242 raise exception_class(message, screen, stacktrace)
243
244 def _value_or_default(self, obj, key, default):
WebDriverException: Message: unknown error: session deleted because of page crash
from unknown error: cannot determine loading status
from tab crashed
(Session info: headless chrome=90.0.4430.212)
And invalid session error:
---------------------------------------------------------------------------
InvalidSessionIdException Traceback (most recent call last)
<ipython-input-38-cf20dd94d732> in <module>
29
---> 30 chart.save('chart.png' scale_factor=3)
c:\programdata\anaconda3\envs\[...]\lib\site-packages\altair\vegalite\v4\api.py in save(self, fp, format, override_data_transformer, scale_factor, vegalite_version, vega_version, vegaembed_version, **kwargs)
474 if override_data_transformer:
475 with data_transformers.disable_max_rows():
--> 476 result = save(**kwds)
477 else:
478 result = save(**kwds)
c:\programdata\anaconda3\envs\[...]\lib\site-packages\altair\utils\save.py in save(chart, fp, vega_version, vegaembed_version, format, mode, vegalite_version, embed_options, json_kwds, webdriver, scale_factor, **kwargs)
119 webdriver=webdriver,
120 scale_factor=scale_factor,
--> 121 **kwargs,
122 )
123 if format == "png":
c:\programdata\anaconda3\envs\[...]\lib\site-packages\altair\utils\mimebundle.py in spec_to_mimebundle(spec, format, mode, vega_version, vegaembed_version, vegalite_version, **kwargs)
58 "see http://github.com/altair-viz/altair_saver/".format(fmt=format)
59 )
---> 60 return altair_saver.render(spec, format, mode=mode, **kwargs)
61 if format == "html":
62 html = spec_to_html(
c:\programdata\anaconda3\envs\[...]\lib\site-packages\altair_saver\_core.py in render(chart, fmts, mode, embed_options, method, **kwargs)
255 Saver = _select_saver(method, mode=mode, fmt=fmt)
256 saver = Saver(spec, mode=mode, embed_options=embed_options, **kwargs)
--> 257 mimebundle.update(saver.mimebundle(fmt))
258
259 return mimebundle
c:\programdata\anaconda3\envs\[...]\lib\site-packages\altair_saver\savers\_saver.py in mimebundle(self, fmts)
88 vegalite_version=self._package_versions["vega-lite"],
89 )
---> 90 bundle[mimetype] = self._serialize(fmt, "mimebundle")
91 return bundle
92
c:\programdata\anaconda3\envs\[...]\lib\site-packages\altair_saver\savers\_selenium.py in _serialize(self, fmt, content_type)
282
283 def _serialize(self, fmt: str, content_type: str) -> MimebundleContent:
--> 284 out = self._extract(fmt)
285 if fmt == "png":
286 assert isinstance(out, str)
c:\programdata\anaconda3\envs\[...]\lib\site-packages\altair_saver\savers\_selenium.py in _extract(self, fmt)
262
263 url = self._serve(html, js_resources)
--> 264 driver.get("about:blank")
265 driver.get(url)
266 try:
c:\programdata\anaconda3\envs\[...]\lib\site-packages\selenium\webdriver\remote\webdriver.py in get(self, url)
331 Loads a web page in the current browser session.
332 """
--> 333 self.execute(Command.GET, {'url': url})
334
335 #property
c:\programdata\anaconda3\envs\[...]\lib\site-packages\selenium\webdriver\remote\webdriver.py in execute(self, driver_command, params)
319 response = self.command_executor.execute(driver_command, params)
320 if response:
--> 321 self.error_handler.check_response(response)
322 response['value'] = self._unwrap_value(
323 response.get('value', None))
c:\programdata\anaconda3\envs\[...]\lib\site-packages\selenium\webdriver\remote\errorhandler.py in check_response(self, response)
240 alert_text = value['alert'].get('text')
241 raise exception_class(message, screen, stacktrace, alert_text)
--> 242 raise exception_class(message, screen, stacktrace)
243
244 def _value_or_default(self, obj, key, default):
InvalidSessionIdException: Message: invalid session id

Python Selenium WebDriverException: Message: chrome not reachable, when both are up to date

suddenly I started getting the WebDriverException error (I assume after my browser updated automatically), and it happened before so I checked my browser version and driver version.
Browser version: 81.0.4044.92 (Official Build) (64-bit)
chromedriver version: 81.0.4044.69
And the error I'm getting is below:
---------------------------------------------------------------------------
WebDriverException Traceback (most recent call last)
<ipython-input-124-def11676151d> in <module>
----> 1 soldStartDate = wait.until(EC.visibility_of_element_located((By.XPATH,"//input[#name='daterangepicker_start']"))).click()
2 soldEndDate = wait.until(EC.visibility_of_element_located((By.XPATH,"//input[#name='daterangepicker_end']"))).click()
3
4 soldStartDate.clear()
5 soldStartDate.send_keys(searchStartDate)
~\AppData\Roaming\Python\Python37\site-packages\selenium\webdriver\support\wait.py in until(self, method, message)
69 while True:
70 try:
---> 71 value = method(self._driver)
72 if value:
73 return value
~\AppData\Roaming\Python\Python37\site-packages\selenium\webdriver\support\expected_conditions.py in __call__(self, driver)
126 def __call__(self, driver):
127 try:
--> 128 return _element_if_visible(_find_element(driver, self.locator))
129 except StaleElementReferenceException:
130 return False
~\AppData\Roaming\Python\Python37\site-packages\selenium\webdriver\support\expected_conditions.py in _find_element(driver, by)
413 raise e
414 except WebDriverException as e:
--> 415 raise e
416
417
~\AppData\Roaming\Python\Python37\site-packages\selenium\webdriver\support\expected_conditions.py in _find_element(driver, by)
409 if thrown."""
410 try:
--> 411 return driver.find_element(*by)
412 except NoSuchElementException as e:
413 raise e
~\AppData\Roaming\Python\Python37\site-packages\selenium\webdriver\remote\webdriver.py in find_element(self, by, value)
976 return self.execute(Command.FIND_ELEMENT, {
977 'using': by,
--> 978 'value': value})['value']
979
980 def find_elements(self, by=By.ID, value=None):
~\AppData\Roaming\Python\Python37\site-packages\selenium\webdriver\remote\webdriver.py in execute(self, driver_command, params)
319 response = self.command_executor.execute(driver_command, params)
320 if response:
--> 321 self.error_handler.check_response(response)
322 response['value'] = self._unwrap_value(
323 response.get('value', None))
~\AppData\Roaming\Python\Python37\site-packages\selenium\webdriver\remote\errorhandler.py in check_response(self, response)
240 alert_text = value['alert'].get('text')
241 raise exception_class(message, screen, stacktrace, alert_text)
--> 242 raise exception_class(message, screen, stacktrace)
243
244 def _value_or_default(self, obj, key, default):
WebDriverException: Message: chrome not reachable
(Session info: chrome=81.0.4044.92)
I have the most recent version of the chrome driver, and I don't understand why I'm getting this error. Any help will be greatly appreciated!

WebDriverException: Message: chrome not reachable When it was working a minute before

I'm writing a scraper using Selenium, and it was working just fine until 15 minutes ago..
All of sudden, I get the following error every time I execute. This is my code:
searchDate = wait.until(EC.element_to_be_clickable((By.XPATH, "/input[#placeholder='select sold date range']")))
searchDate.click()
time.sleep(5)
And the error:
WebDriverException: Message: chrome not reachable
(Session info: chrome=80.0.3987.163)
And my chrome setting:
options = webdriver.ChromeOptions()
prefs = {'download.default_directory': new_dir}
options.add_experimental_option('prefs', prefs)
browser = webdriver.Chrome(executable_path = r'C:/Program Files/chromedriver/chromedriver.exe')
I tried to solve the problem by chain the options as other post suggested, such as:
options = webdriver.ChromeOptions()
options.add_argument('--no-sandbox')
prefs = {'download.default_directory': new_dir}
options.add_experimental_option('prefs', prefs)
browser = webdriver.Chrome(executable_path = r'C:/Program Files/chromedriver/chromedriver.exe', chrome_options=options)
But I still get the exact same error. Why did it happen?
Any advice will be greatly appreciated!
++
---------------------------------------------------------------------------
WebDriverException Traceback (most recent call last)
<ipython-input-102-c7d533e1df88> in <module>
----> 1 wait.until(EC.visibility_of_element_located((By.XPATH,"//input[#name='daterangepicker_start']"))).click()
2
3
~\AppData\Roaming\Python\Python37\site-packages\selenium\webdriver\support\wait.py in until(self, method, message)
69 while True:
70 try:
---> 71 value = method(self._driver)
72 if value:
73 return value
~\AppData\Roaming\Python\Python37\site-packages\selenium\webdriver\support\expected_conditions.py in __call__(self, driver)
126 def __call__(self, driver):
127 try:
--> 128 return _element_if_visible(_find_element(driver, self.locator))
129 except StaleElementReferenceException:
130 return False
~\AppData\Roaming\Python\Python37\site-packages\selenium\webdriver\support\expected_conditions.py in _find_element(driver, by)
413 raise e
414 except WebDriverException as e:
--> 415 raise e
416
417
~\AppData\Roaming\Python\Python37\site-packages\selenium\webdriver\support\expected_conditions.py in _find_element(driver, by)
409 if thrown."""
410 try:
--> 411 return driver.find_element(*by)
412 except NoSuchElementException as e:
413 raise e
~\AppData\Roaming\Python\Python37\site-packages\selenium\webdriver\remote\webdriver.py in find_element(self, by, value)
976 return self.execute(Command.FIND_ELEMENT, {
977 'using': by,
--> 978 'value': value})['value']
979
980 def find_elements(self, by=By.ID, value=None):
~\AppData\Roaming\Python\Python37\site-packages\selenium\webdriver\remote\webdriver.py in execute(self, driver_command, params)
319 response = self.command_executor.execute(driver_command, params)
320 if response:
--> 321 self.error_handler.check_response(response)
322 response['value'] = self._unwrap_value(
323 response.get('value', None))
~\AppData\Roaming\Python\Python37\site-packages\selenium\webdriver\remote\errorhandler.py in check_response(self, response)
240 alert_text = value['alert'].get('text')
241 raise exception_class(message, screen, stacktrace, alert_text)
--> 242 raise exception_class(message, screen, stacktrace)
243
244 def _value_or_default(self, obj, key, default):
WebDriverException: Message: chrome not reachable
(Session info: chrome=81.0.4044.92)
Can you please check if your chrome browser is compatible with your chrome browser?
Could be possible your chrome is autommatically updated and now your chrome browser is not compatible with chrome driver.
You can download chrome driver from here

How to use the Selenium WebDriver findElement (By.xpath ())?

please, I'm trying to get information from a public website of the Brazilian congress: name of a voting session, date and table with list of votes
This site: http://www2.camara.leg.br/atividade-legislativa/plenario/chamadaExterna.html?link=http://www.camara.gov.br/internet/votacao/mostraVotacao.asp?ideVotacao=6706&tipo=partido
I used Python 3, selenium webdriver and PhantomJS:
from selenium import webdriver
path_to_phantomjs = '/Users/George/Documents/phantomjs/phantomjs-2.1.1-windows/bin/phantomjs'
browser = webdriver.PhantomJS(executable_path = path_to_phantomjs)
browser.get("http://www2.camara.leg.br/atividade-legislativa/plenario/chamadaExterna.html?link=http://www.camara.gov.br/internet/votacao/mostraVotacao.asp?ideVotacao=6706&tipo=partido")
nome_votacao = browser.find_element_by_xpath("//[#id='corpoVotacao']/p[3]/text()")
data_votacao = browser.find_element_by_xpath("//[#id='corpoVotacao']/div[1]/div[1]/div/div/p[1]/text()[1]")
list_deputados = browser.find_elements_by_xpath(".//table[#class='tabela-2']")
But it looks like I'm wrongly selecting locations
The nome_votacao appears this error message:
---------------------------------------------------------------------------
InvalidSelectorException Traceback (most recent call last)
<ipython-input-11-e67933637ae0> in <module>()
----> 1 nome_votacao = browser.find_element_by_xpath("//[#id='corpoVotacao']/p[3]/text()")
c:\users\george\appdata\local\programs\python\python36-32\code\votos\lib\site-packages\selenium\webdriver\remote\webdriver.py in find_element_by_xpath(self, xpath)
363 driver.find_element_by_xpath('//div/td[1]')
364 """
--> 365 return self.find_element(by=By.XPATH, value=xpath)
366
367 def find_elements_by_xpath(self, xpath):
c:\users\george\appdata\local\programs\python\python36-32\code\votos\lib\site-packages\selenium\webdriver\remote\webdriver.py in find_element(self, by, value)
841 return self.execute(Command.FIND_ELEMENT, {
842 'using': by,
--> 843 'value': value})['value']
844
845 def find_elements(self, by=By.ID, value=None):
c:\users\george\appdata\local\programs\python\python36-32\code\votos\lib\site-packages\selenium\webdriver\remote\webdriver.py in execute(self, driver_command, params)
306 response = self.command_executor.execute(driver_command, params)
307 if response:
--> 308 self.error_handler.check_response(response)
309 response['value'] = self._unwrap_value(
310 response.get('value', None))
c:\users\george\appdata\local\programs\python\python36-32\code\votos\lib\site-packages\selenium\webdriver\remote\errorhandler.py in check_response(self, response)
192 elif exception_class == UnexpectedAlertPresentException and 'alert' in value:
193 raise exception_class(message, screen, stacktrace, value['alert'].get('text'))
--> 194 raise exception_class(message, screen, stacktrace)
195
196 def _value_or_default(self, obj, key, default):
InvalidSelectorException: Message: {"errorMessage":"Unable to locate an element with the xpath expression //[#id='corpoVotacao']/p[3]/text() because of the following error:\nError: INVALID_EXPRESSION_ERR: DOM XPath Exception 51","request":{"headers":{"Accept":"application/json","Accept-Encoding":"identity","Connection":"close","Content-Length":"118","Content-Type":"application/json;charset=UTF-8","Host":"127.0.0.1:55799","User-Agent":"Python http auth"},"httpVersion":"1.1","method":"POST","post":"{\"using\": \"xpath\", \"value\": \"//[#id='corpoVotacao']/p[3]/text()\", \"sessionId\": \"366665f0-be24-11e7-aa25-75da268b98e2\"}","url":"/element","urlParsed":{"anchor":"","query":"","file":"element","directory":"/","path":"/element","relative":"/element","port":"","host":"","password":"","user":"","userInfo":"","authority":"","protocol":"","source":"/element","queryKey":{},"chunks":["element"]},"urlOriginal":"/session/366665f0-be24-11e7-aa25-75da268b98e2/element"}}
Screenshot: available via screen
The data_votacao this error message:
---------------------------------------------------------------------------
InvalidSelectorException Traceback (most recent call last)
<ipython-input-12-24c43341f310> in <module>()
----> 1 data_votacao = browser.find_element_by_xpath("//[#id='corpoVotacao']/div[1]/div[1]/div/div/p[1]/text()[1]")
c:\users\george\appdata\local\programs\python\python36-32\code\votos\lib\site-packages\selenium\webdriver\remote\webdriver.py in find_element_by_xpath(self, xpath)
363 driver.find_element_by_xpath('//div/td[1]')
364 """
--> 365 return self.find_element(by=By.XPATH, value=xpath)
366
367 def find_elements_by_xpath(self, xpath):
c:\users\george\appdata\local\programs\python\python36-32\code\votos\lib\site-packages\selenium\webdriver\remote\webdriver.py in find_element(self, by, value)
841 return self.execute(Command.FIND_ELEMENT, {
842 'using': by,
--> 843 'value': value})['value']
844
845 def find_elements(self, by=By.ID, value=None):
c:\users\george\appdata\local\programs\python\python36-32\code\votos\lib\site-packages\selenium\webdriver\remote\webdriver.py in execute(self, driver_command, params)
306 response = self.command_executor.execute(driver_command, params)
307 if response:
--> 308 self.error_handler.check_response(response)
309 response['value'] = self._unwrap_value(
310 response.get('value', None))
c:\users\george\appdata\local\programs\python\python36-32\code\votos\lib\site-packages\selenium\webdriver\remote\errorhandler.py in check_response(self, response)
192 elif exception_class == UnexpectedAlertPresentException and 'alert' in value:
193 raise exception_class(message, screen, stacktrace, value['alert'].get('text'))
--> 194 raise exception_class(message, screen, stacktrace)
195
196 def _value_or_default(self, obj, key, default):
InvalidSelectorException: Message: {"errorMessage":"Unable to locate an element with the xpath expression //[#id='corpoVotacao']/div[1]/div[1]/div/div/p[1]/text()[1] because of the following error:\nError: INVALID_EXPRESSION_ERR: DOM XPath Exception 51","request":{"headers":{"Accept":"application/json","Accept-Encoding":"identity","Connection":"close","Content-Length":"143","Content-Type":"application/json;charset=UTF-8","Host":"127.0.0.1:55799","User-Agent":"Python http auth"},"httpVersion":"1.1","method":"POST","post":"{\"using\": \"xpath\", \"value\": \"//[#id='corpoVotacao']/div[1]/div[1]/div/div/p[1]/text()[1]\", \"sessionId\": \"366665f0-be24-11e7-aa25-75da268b98e2\"}","url":"/element","urlParsed":{"anchor":"","query":"","file":"element","directory":"/","path":"/element","relative":"/element","port":"","host":"","password":"","user":"","userInfo":"","authority":"","protocol":"","source":"/element","queryKey":{},"chunks":["element"]},"urlOriginal":"/session/366665f0-be24-11e7-aa25-75da268b98e2/element"}}
Screenshot: available via screen
And the list_deputados generates an empty list
The data_votacao was to have this content: "10/11/2015 20:15". And it has this XPath in the inspect: //*[#id="corpoVotacao"]/p[2]/text()[1]
The nome_votação was to have this content: "MPV Nº 688/2015 - PROJETO DE LEI DE CONVERSÃO - Nominal Eletrônica". And it has this XPath in the inspect: //*[#id="corpoVotacao"]/p[3]/text()
And the list_deputados was to have the complete table with the votes (names, UF and vote), starting in the line "Parlamentar". It has this XPath: //*[#id="listagem"]/table. And the class="tabela-2"
Does anyone know the correct form of the commands? Or some tutorial?
The first problem is that all the interested elements are inside an iframe.
So, first you have to switch to the iframe:
iframe = driver.find_element_by_xpath("//iframe[#id='ifr']")
driver.switch_to_frame(iframe)
Also the xpath aren't correct. You could use the following:
nome_votacao = driver.find_element_by_xpath("//*[#id='corpoVotacao']/p[3]")
data_votacao = driver.find_element_by_xpath("//*[#id='corpoVotacao']/div[1]/div[1]/div/div/p[1]")
list_deputados = driver.find_elements_by_xpath("*//div[#id='listagem']/table[#class='tabela-2']/tbody/tr/td")
The entire code:
driver = webdriver.PhantomJS(executable_path=r'/pathTo/phantomjs')
driver.get("http://www2.camara.leg.br/atividade-legislativa/plenario/chamadaExterna.html?link=http://www.camara.gov.br/internet/votacao/mostraVotacao.asp?ideVotacao=6706&tipo=partido")
iframe = driver.find_element_by_xpath("//iframe[#id='ifr']")
driver.switch_to_frame(iframe)
nome_votacao = driver.find_element_by_xpath("//*[#id='corpoVotacao']/p[3]")
data_votacao = driver.find_element_by_xpath("//*[#id='corpoVotacao']/div[1]/div[1]/div/div/p[1]")
list_deputados = driver.find_elements_by_xpath("*//div[#id='listagem']/table[#class='tabela-2']/tbody/tr/td")
print(nome_votacao.text)
print("----------------------------")
print(data_votacao.text)
print("----------------------------")
i=0
maxLen=len(list_deputados)
print("maxLen: " + str(maxLen))
for deputado in list_deputados:
print("Parlamentar DEM: " + list_deputados[i].text)
print("UF: " + list_deputados[i+1].text)
print("Voto: " + list_deputados[i+2].text)
i=i+2
print("--------------"+str(i)+"-------------")
if(i==maxLen-3): #don't consider the last row: Total Solidaried: 11
break
driver.close()
Can you try
//*[#id="corpoVotacao"]/p[3]/text() as XPATH instead of
//[#id='corpoVotacao']/p[3]/text()
//*[#id="corpoVotacao"]/div[1]/div[1]/div/div/p[1]/text()[1] as XPATH instead of
//[#id='corpoVotacao']/div[1]/div[1]/div/div/p[1]/text()[1]
last XPATH seems correct to me, also can you have implicit wait after get() just to ensure elements are loaded on UI.

Categories

Resources