I am trying to extract all those tags whose class name fits the regex pattern frag-0-0, frag-1-0, etc. from this enter link description here
I am trying the following code
driver = webdriver.PhantomJS()
for frg in frgs:
driver.get(URL + frg[1:])
frags=driver.find_elements_by_xpath("//*[starts-with(#id, 'frag-') and ends-with(#id, '-0')]")
for frag in frags:
for tag in frag.find_elements_by_css_selector('[class^=fragmark]'):
lst.append([tag.get_attribute('class'), tag.text])
driver.quit()
This is my traceback:
Traceback (most recent call last):
File "/home/ubuntu/workspace/vroniplag/vroni.py", line 116, in
op('Aaf')
File "/home/ubuntu/workspace/vroniplag/vroni.py", line 101, in op
plags=getplags(cd)
File "/home/ubuntu/workspace/vroniplag/vroni.py", line 92, in getplags
frags=driver.find_elements_by_xpath("//[starts-with(#id, 'frag-') and ends-with(#id, '-0')]")
File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/webdriver.py",
line 305, in find_elements_by_xpath
return self.find_elements(by=By.XPATH, value=xpath)
File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/webdriver.py",
line 778, in find_elements
'value': value})['value']
File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/webdriver.py",
line 236, in execute
self.error_handler.check_response(response)
File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/errorhandler.py",
line 192, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.InvalidSelectorException: Message: Error Message => 'Unable to locate an element with the xpath
expression //[starts-with(#id, 'frag-') and ends-with(#id, '-0')]
because of the following error:
Error: INVALID_EXPRESSION_ERR: DOM XPath Exception 51'
caused by Request => {"headers":{"Accept":"application/json","Accept-Encoding":"identity","Connection":"close","Content-Length":"139","Content-Type":"application/json;charset=UTF-8","Host":"127.0.0.1:45340","User-Agent":"Python-urllib/2.7"},"httpVersion":"1.1","method":"POST","post":"{\"using\":
\"xpath\", \"sessionId\": \"0dbc6ad0-4352-11e6-8cb8-4faebd646180\",
\"value\": \"//*[starts-with(#id, 'frag-') and ends-with(#id,
'-0')]\"}","url":"/elements","urlParsed":{"anchor":"","query":"","file":"elements","directory":"/","path":"/elements","relative":"/elements","port":"","host":"","password":"","user":"","userInfo":"","authority":"","protocol":"","source":"/elements","queryKey":{},"chunks":["elements"]},"urlOriginal":"/session/0dbc6ad0-4352-11e6-8cb8-4faebd646180/elements"}
Screenshot: available via screen
What am I doing wrong?
You can try to replace
"//*[starts-with(#id, 'frag-') and ends-with(#id, '-0')]"
with
"//*[starts-with(#id, 'frag-') and contains(#id, '-0')]"
as Selenium doesn't support ends-with option
Related
I have a for loop that iterates over a list of animal names from the internal center database. However, some of the animals are missing tables. I have been able to bypass this issue using Seleniums NoSuchElementException.
s = '''\
Animal Number {animal_num} Not Found!
Please be sure you entered a valid animal ID.\
'''.format(animal_num=i)
# Test for internal server error
try:
driver.find_element_by_name("query_input").send_keys(i)
driver.find_element_by_name("submit").click()
except NoSuchElementException:
print(s)
continue
However, at least one id that throws a javascript error that NoSuchElementException cant catch.
Traceback (most recent call last):
File ".\webvitals.py", line 77, in <module>
diarrhea_observations.diarrhea_observations(driver, query_list)
File "C:\Users\Noah Siegel\Desktop\webvitals\scripts\diarrhea_observations.py", line 32, in diarrhea_observations
driver.find_element_by_name("query_input").send_keys(i)
File "C:\Users\Noah Siegel\.conda\envs\webvitals\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 487, in find_element_by_name
return self.find_element(by=By.NAME, value=name)
File "C:\Users\Noah Siegel\.conda\envs\webvitals\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 955, in find_element
'value': value})['value']
File "C:\Users\Noah Siegel\.conda\envs\webvitals\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 312, in execute
self.error_handler.check_response(response)
File "C:\Users\Noah Siegel\.conda\envs\webvitals\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: Unable to locate element: [name="query_input"]
When I manually enter the animal id in a browser I am taken to this page
Here is the code which I use to open the new tab using Selenium Python,
self.driver.execute_script("window.open(arguments[0])", self.listing_url)
self.driver.switch_to.window(self.driver.window_handles[1])
time.sleep(10)
and it works perfectly. But I am planning to use WebDriverWait instead of time.sleep
self.driver.execute_script("window.open(arguments[0])", self.listing_url)
self.driver.switch_to.window(self.driver.window_handles[1])
WebDriverWait(self.driver, 20).until(EC.presence_of_element_located((By.XPATH, '//div[#class="honbun"]')))
and when I am using it, it's throwing an error. Here is the error log:
Traceback (most recent call last):
File "D:\tender_scraper\engine\Spider.py", line 308, in parse_index_page
index_handle = self.load_listing_page(listing, i, index_listing)
File "D:\tender_scraper\spiders\gunmapref_listing_spider.py", line 298, in load_listing_page
WebDriverWait(self.driver, 20).until(EC.presence_of_element_located((By.XPATH, '//div[#class="honbun"]')))
File "D:\scraper\lib\site-packages\selenium\webdriver\support\wait.py", line 78, in until
value = method(self._driver)
File "D:\scraper\lib\site-packages\selenium\webdriver\support\expected_conditions.py", line 64, in _predicate
return driver.find_element(*locator)
File "D:\scraper\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 1251, in find_element
return self.execute(Command.FIND_ELEMENT, {
File "D:\scraper\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 430, in execute
self.error_handler.check_response(response)
File "D:\scraper\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 247, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.WebDriverException: Message: unknown error: missing or invalid columnNumber
(Session info: headless chrome=104.0.5112.48)
Any idea what's wrong with it & how to solve it? Previously, I used chrome 103 and changed it to 104 but still got the same error. It seems the error only occurs when changing into the new tab.
I need to edit this file with any python module:
https://1drv.ms/x/s!Aq2lKlhSyqf5mjiC6CYkC1QGXNSy?e=Wh88AT
but I don't want to download it edit and then upload again. I tried using webbot, but I couldn't even find the 'editing mode' button.
import time
from time import sleep
from webbot import Browser
driver = Browser()
driver.new_tab(url='https://1drv.ms/x/s!Aq2lKlhSyqf5mjiC6CYkC1QGXNSy?e=Wh88AT')
driver.switch_to_tab(1)
driver.close_current_tab()
print("waiting...")
sleep(20)
print("start")
driver.click(classname = 'list-349' , tag = 'button' )
but it returns this:
Traceback (most recent call last):
File "main.py", line 19, in <module>
driver.click(classname = 'list-349' , tag = 'button' )
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/webbot/webbot.py", line 466, in click
maxElements = self.__find_element(text, tag, classname, id, number, css_selector, xpath, loose_match)
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/webbot/webbot.py", line 294, in __find_element
element_fetch_helper(("//body//{}".format(tag)), score=50)
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/webbot/webbot.py", line 202, in element_fetch_helper
add_to_init_text_matches_score(self.driver.find_elements_by_xpath(xpath), score)
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 410, in find_elements_by_xpath
return self.find_elements(by=By.XPATH, value=xpath)
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 1005, in find_elements
return self.execute(Command.FIND_ELEMENTS, {
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "/opt/virtualenvs/python3/lib/python3.8/site-packages/selenium/webdriver/remote/errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchWindowException: Message: no such window: target window already closed
from unknown error: web view not found
(Session info: chrome=91.0.4472.101)
(Driver info: chromedriver=2.39.562737 (dba483cee6a5f15e2e2d73df16968ab10b38a2bf),platform=Linux 5.15.0-1012-gcp x86_64)
I'm using replit by the way.
I followed this tutorial for Selenium with Python:
So I did the same as the tutorial:
from selenium import webdriver
driver=webdriver.Chrome()
driver.get('https://www.youtube.com/')
searchbox = driver.find_element_by_xpath('//*[#id="search-input"]')
searchbox.send_keys('cooking')
But I get this error:
Traceback (most recent call last):
File "C:/ANGUS/DATA_ANALYSIS/WEBSCRAPING/TextXXX.py", line 7, in <module>
searchbox.send_keys('cooking')
File "C:\Users\Angus\PycharmProjects\Test\venv\lib\site-packages\selenium\webdriver\remote\webelement.py", line 477, in send_keys
self._execute(Command.SEND_KEYS_TO_ELEMENT,
File "C:\Users\Angus\PycharmProjects\Test\venv\lib\site-packages\selenium\webdriver\remote\webelement.py", line 633, in _execute
return self._parent.execute(command, params)
File "C:\Users\Angus\PycharmProjects\Test\venv\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "C:\Users\Angus\PycharmProjects\Test\venv\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.ElementNotInteractableException: Message: element not interactable
(Session info: chrome=81.0.4044.138)
How can I fix this?
Another way to do that in order not to use the search is by inserting the query itself.
Code:
your_query = "cooking"
driver.get("https://www.youtube.com/results?search_query=" + str(your_query))
time.sleep(5)
In this way, it is like you are going immediately in the output that you want after searching a specific query of yours. So, let's say that instead of typing cooking, you can .get() to the URL you want by adding your query into this URL.
I m trying to open URLS From TXT file with selenium webdriver.But code is giving me error.Here is code that i m using with python 3.4.3.What can be the problem in this code?
from selenium import webdriver
with open("vps.txt","r") as fi:
urls = fi.read().splitlines()
for url in urls:
browser = webdriver.Firefox()
browser.get(url)
I get this error:
Traceback (most recent call last):
File "C:\pselenium\browse.py", line 8, in <module>
browser.get(url)
File "C:\Python34\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 245, in get
self.execute(Command.GET, {'url': url})
File "C:\Python34\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 233, in execute
self.error_handler.check_response(response)
File "C:\Python34\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 194, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.WebDriverException: Message: Target URL www.youtube.com/watch?v=DowtrQhvbUM is not well-formed.
Stacktrace:
at FirefoxDriver.prototype.get (file:///C:/Users/naV/AppData/Local/Temp/tmpja7hed7p/extensions/fxdriver#googlecode.com/components/driver-component.js:10636)
at DelayedCommand.prototype.executeInternal_/h (file:///C:/Users/naV/AppData/Local/Temp/tmpja7hed7p/extensions/fxdriver#googlecode.com/components/command-processor.js:12661)
at DelayedCommand.prototype.executeInternal_ (file:///C:/Users/naV/AppData/Local/Temp/tmpja7hed7p/extensions/fxdriver#googlecode.com/components/command-processor.js:12666)
at DelayedCommand.prototype.execute/< (file:///C:/Users/naV/AppData/Local/Temp/tmpja7hed7p/extensions/fxdriver#googlecode.com/components/command-processor.js:12608)
Target URL www.youtube.com/watch?v=DowtrQhvbUM is not well formed
It looks like you've got a problem with your file encoding. I'm not sure if that's the BOM or not, but you should check your file with a hex editor.