Develop Reference

Unable to locate element by class name using selenium via Python. Why so?

I wrote the following code in order to scrape the text of the element <h3 class="h4 mb-10">Total nodes: 1,587</h3>
from https://blockchair.com/dogecoin/nodes.
#!/usr/bin/python3
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
path = "/usr/local/bin/chromedriver"
driver = webdriver.Chrome(path)
driver.get("https://blockchair.com/dogecoin/nodes")
def scraping_fnd():
try:
#nodes = driver.find_element_by_class_name("h4 mb-10")
#NoSuchElementException: Message: no such element: Unable to locate element: {"method":"css selector","selector":".h4 mb-10"}. There are 3 elements pf this class
#nodes = WebDriverWait(driver, 10).until(expected_conditions.presence_of_element_located((By.CLASS_NAME, "h4 mb-10")))#selenium.common.exceptions.TimeoutException: Message:
nodes = WebDriverWait(driver, 10).until(expected_conditions.presence_of_element_located((By.CSS_SELECTOR, ".h4 mb-10")))#selenium.common.exceptions.TimeoutException: Message:
nodes = nodes.text
print(nodes)
finally:
driver.quit()#Closes the tab even when return is executed
scraping_fnd()
I'm aware that there are perhaps less bloated options than selenium to scrape the target in question, yet the said code is just a snippet,
a part of a more extensive script that relies on selenium for its other tasks. Thus let us limit the scope of the answers to selenium
only.
Although there are three elements of the class "h4 mb-10" on the page, I am unable to locate the element. When I call driver.find_element_by_class_name("h4 mb-10"), I get:
Traceback (most recent call last):
File "./protocols.py", line 34, in <module>
scraping_fnd()
File "./protocols.py", line 20, in scraping_fnd
nodes = driver.find_element_by_class_name("h4 mb-10")#(f"//span[#title = \"{name}\"]")
File "/home/jerzy/.local/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 564, in find_element_by_class_name
return self.find_element(by=By.CLASS_NAME, value=name)
File "/home/jerzy/.local/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 976, in find_element
return self.execute(Command.FIND_ELEMENT, {
File "/home/jerzy/.local/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "/home/jerzy/.local/lib/python3.8/site-packages/selenium/webdriver/remote/errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"css selector","selector":".h4 mb-10"}
(Session info: chrome=90.0.4430.212)
XP
Applying waits, currently commented out in the snippet, was to no avail. I came across this question and so I tried calling WebDriverWait(driver, 10).until(expected_conditions.presence_of_element_located((By.CSS_SELECTOR, ".h4 mb-10"))).
I got :
Traceback (most recent call last):
File "./protocols.py", line 33, in <module>
scraping_fnd()
File "./protocols.py", line 23, in scraping_fnd
nodes = WebDriverWait(driver, 10).until(expected_conditions.presence_of_element_located((By.CSS_SELECTOR, ".h4 mb-10")))#selenium.common.exceptions.TimeoutException: Message:
File "/home/jerzy/.local/lib/python3.8/site-packages/selenium/webdriver/support/wait.py", line 80, in until
raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message:
I have no clue what am I doing wrong. Is it doable to scrape the target with selenium without using Xpaths?

Try with this xpath. Find all the elements with this xpath and with indexing extract the required Text.
//div[#class='nodes_chain']/div[1]/h3 # for "Total nodes" option
//div[#class='nodes_chain']//h3 # for all the option.
from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
driver = webdriver.Chrome(executable_path="path to chromedriver.exe")
driver.maximize_window()
driver.implicitly_wait(10)
driver.get("https://blockchair.com/dogecoin/nodes")
wait = WebDriverWait(driver,30)
option = wait.until(EC.presence_of_element_located((By.XPATH,"//div[#class='nodes_chain']/div[1]/h3")))
print(option.text)
options = wait.until(EC.presence_of_all_elements_located((By.XPATH,"//div[#class='nodes_chain']//h3")))
# First option
print(options[0].text)
# All the options
for opt in options:
print(opt.text)
Total nodes: 1,562
Total nodes: 1,562
Total nodes: 1,562
Node versions
Block heights

If you want to go with css selector it would be
.h4.mb-10
You can go with class name
If either one uniquely identifies the element or xpath
//h3[#class="h4 mb-10"]

Try the below (Note that selenium is not involved in the solution)
import requests
from bs4 import BeautifulSoup
r = requests.get('https://blockchair.com/dogecoin/nodes')
soup = BeautifulSoup(r.content, 'html.parser')
lst = soup.find_all('h3',{"class": "h4 mb-10"})
lst = [h for h in lst if 'Total' in h.text]
print(lst[0].text)
output
Total nodes: 1,593

Click on the attribut of a tag with the librarie selenium

I trying to click on the attribute class container in the tag div with a librarie Selenium. Here is code :
from selenium import webdriver
from selenium.webdriver.common.by import By
driver = webdriver.Chrome()
driver.get('https://www.flashscore.com/')
driver.find_element(By.CSS_SELECTOR, ".header__button header__button--search").click()
And here is error display :
>>> driver.find_element(By.CSS_SELECTOR, ".header__button header__button--search").click();
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "C:\Users\avis\AppData\Local\Programs\Python\Python38-32\lib\site-packages\selenium\webdriver\remote\webdriver.py",
line 976, in find_element
return self.execute(Command.FIND_ELEMENT, {
File "C:\Users\avis\AppData\Local\Programs\Python\Python38-32\lib\site-packages\selenium\webdriver\remote\webdriver.py",
line 321, in execute
self.error_handler.check_response(response)
File "C:\Users\avis\AppData\Local\Programs\Python\Python38-32\lib\site-packages\selenium\webdriver\remote\errorhandler.py",
line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"css
selector","selector":".header__button header__button--search"}
(Session info: chrome=85.0.4183.102)
the code is inspired of the doc of selenium : https://www.selenium.dev/documentation/en/getting_started_with_webdriver/performing_actions_on_the_aut/
I did some research and find a function which makes it possible to overcome the exception and it'is :element_to_be_clickable()
it is used to wait as the element is display and be clickable, according to the doc.
And I used it as this :
> > element = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CLASS_NAME, "header__button
> header__button--search"))
> element.click();
But this syntaxe error is display in console :
File "", line 2
element.click()
^ SyntaxError: invalid syntax
while I not see of syntax error.
From where can arise from the error ?
And function use is good ?

In HTML, you specify multiple classes with a space:
<span class="class1 class2">....</span>
In selenium, when searching for a single class, just use one of the class names:
driver.find_element(By.CSS_SELECTOR, ".class1")
This will code will open the site with Selenium and click the Search button:
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium import webdriver
# prevent version errors and plugin warning, may not be needed for you
options = webdriver.ChromeOptions()
options.add_argument("disable-extensions")
options.add_argument("disable-plugins")
options.experimental_options["useAutomationExtension"] = False # prevent load error - Error Loading Extension - Failed to load extension from ... - Could not load extension from ... Loading of unpacked extensions is disabled
driver = webdriver.Chrome(ChromeDriverManager().install(), options=options)
# main code
driver.get('https://www.flashscore.com/')
element = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, ".header__button--search")))
driver.find_element(By.CSS_SELECTOR, ".header__button--search").click()
To find an element using multiple classes, check this post:
Find div element by multiple class names?

How to Access span element and change it's value

I am trying to create a python + selenium script in order to fetch CSV from a salesforce pardot page.
Can Somebody please help me in accessing the nested span element inside the dropdown list which will be generated on button click.
I am adding my code done so far and I am getting Timeout Error While running my script.
from selenium import webdriver
import time
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium import webdriver
try:
chrome_options = webdriver.ChromeOptions()
prefs = {'download.default_directory': r'C:\Pardot'}
chrome_options.add_experimental_option('prefs', prefs)
driver = webdriver.Chrome(executable_path="D:\XXX XXXX\XXXX\drivers\chromedriver.exe", options=chrome_options)
driver.get('https://pi.pardot.com/engagementStudio/studio#/15627/reporting')
user_name = driver.find_element_by_css_selector('#email_address')
user_name.send_keys('XXXXXXXXXXXXXXXXXXX')
password = driver.find_element_by_css_selector('#password')
password.send_keys('XXXXXXXXXXXXXXXXX)
submit_button = driver.find_element_by_css_selector('input.btn')
submit_button.click()
WebDriverWait(driver, 10).until(
EC.frame_to_be_available_and_switch_to_it((By.CSS_SELECTOR, "iframe#content-frame")))
WebDriverWait(driver, 10).until(EC.text_to_be_present_in_element_value(text_='All Time',locator=(By.CSS_SELECTOR,"span[data-qa='reporting-filter-trigger-value']")))
except (RuntimeError) as e:
print(e)
finally:
time.sleep(10)
driver.close()
Screenshot for span DOM element
Error Stack trace:
C:\Users\Vipul.Garg\AppData\Local\Programs\Python\Python37\python.exe "D:/Vipul Garg Backup/XXXX/testingCSVfetching.py"
Traceback (most recent call last):
File "D:/Vipul Garg Backup/XXXX/testingCSVfetching.py", line 22, in <module>
WebDriverWait(driver, 10).until(EC.text_to_be_present_in_element_value(text_='All Time',locator=(By.CSS_SELECTOR,"span[data-qa='reporting-filter-trigger-value']")))
File "C:\Users\Vipul.Garg\AppData\Local\Programs\Python\Python37\lib\site-packages\selenium\webdriver\support\wait.py", line 80, in until
raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message:

try this approach
spans = driver.find_element_by_css_selector('.slds-button_reset.slds-grow.slds-has-blur-focus.trigger-button').get_attribute('innerHTML')
print(spans)
You should be getting all span.

login into to a website using selenium python

I am trying to scrape some information from the below website. I am new to this so dont fully understand what is happening. I have basically been sourcing information from various examples i have found on internet
I am using selenium and python to help me navigate to the page from where i can scrape some information
I have used the below script
I am able to navigate to the home page , close cookies and click on sign in .
at this point a pop up opens up for entering user id and password
the div element is not being identified , each time i get error saying no such element is present
i added the wait with expected conditions , however i get the below error
C:\Users\user\AppData\Local\Programs\Python\Python38-32\SeleniumWebscraper2.py:14: DeprecationWarning: use options instead of chrome_options
browser = webdriver.Chrome(executable_path="C:/Users/user/Downloads/chromedriver_win32/chromedriver.exe", chrome_options=chrome_options)
DevTools listening on ws://127.0.0.1:2672/devtools/browser/f6aca700-5569-4367-9ceb-71e88fcc3082
CDwindow-BD7440D2580236DB9EEFE8B8FE1730D6
0
Traceback (most recent call last):
File "C:\Users\user\AppData\Local\Programs\Python\Python38-32\SeleniumWebscraper2.py", line 37, in
browser.find_element_by_xpath("//button[#class='gigya-input-text']").send_keys('x14127032#student.ncirl.ie')
File "C:\Users\user\AppData\Local\Programs\Python\Python38-32\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 394, in find_element_by_xpath
return self.find_element(by=By.XPATH, value=xpath)
File "C:\Users\user\AppData\Local\Programs\Python\Python38-32\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 976, in find_element
return self.execute(Command.FIND_ELEMENT, {
File "C:\Users\user\AppData\Local\Programs\Python\Python38-32\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "C:\Users\user\AppData\Local\Programs\Python\Python38-32\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"xpath","selector":"//button[#class='gigya-input-text']"}
(Session info: chrome=78.0.3904.97)
https://education.independent.ie/league
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
import requests
from bs4 import BeautifulSoup
chrome_options = webdriver.ChromeOptions()
chrome_options.add_experimental_option("prefs", {"profile.default_content_settings.cookies": 2})
browser = webdriver.Chrome(executable_path="C:/Users/user/Downloads/chromedriver_win32/chromedriver.exe", chrome_options=chrome_options)
browser.get("https://education.independent.ie/league/school/abbey-community-college-roscommon-558")
browser.implicitly_wait(10) # seconds
selector = ".gigya-input-text"
main_window_handle = None
while not main_window_handle:
main_window_handle = browser.current_window_handle
print(main_window_handle)
browser.find_element_by_xpath("//button[#class='qc-cmp-button']").click()
WebDriverWait(browser, 10)
browser.switch_to.window(browser.current_window_handle)
browser.find_element_by_xpath("//button[#class='button-b gigya-sign-in']").send_keys('\n')
WebDriverWait(browser, 120).until(EC.visibility_of_element_located((By.CSS_SELECTOR, selector )))
print(len(browser.find_elements_by_id("gigya-input-text")))
browser.find_element_by_xpath("//button[#class='gigya-input-text']").send_keys('x14127032#student.ncirl.ie')

try this:
# import web driver
from selenium import webdriver
# specifies the path to the chromedriver.exe
driver = webdriver.Chrome("/opt/anaconda2/chromedriver")
# driver.get method() will navigate to a page given by the URL address
driver.get("https://www.linkedin.com/login?")
# locate email form by_name
username = driver.find_element_by_name('session_key')
# locate email form by_xpath
#username = driver.find_element_by_xpath('//*[#id="username"]')
# locate email form by_id
#username = driver.find_element_by_id("username")
#send_keys() to simulate key strokes
username.send_keys("username#email.com")
# locate password form by_name
#password = driver.find_element_by_name('session_password')
#locate password form id
#password = driver.find_element_by_id("password")
#locate password form by_xpath
password = driver.find_element_by_xpath('//*[#id="password"]')
# send_keys() to simulate key strokes
password.send_keys("password")
# locate submit button by_class_name
log_in_button = driver.find_element_by_class_name('btn__primary--large')
# locate submit button by_xpath
#log_in_button = driver.find_element_by_xpath('//*[#type="submit"]')
# .click() to mimic button click
log_in_button.click()

You are getting this error because the part of id gig_1574604202288_showScreenSet that your are trying to find is random. You can use other selector to locate this element e.g. (By.CSS_SELECTOR, "div[id*='showScreenSet']"). You can read about this more here and practice here.
Also WebDriverWait(browser, 120) does nothing. If you want global wait you can use implicit wait. Read about this here.

Element not interactable (NG-click) Selenium Python .click() in a DIV [duplicate]

This question already has answers here:
How to resolve ElementNotInteractableException: Element is not visible in Selenium webdriver?
(6 answers)
Selenium WebDriver throws Exception in thread "main" org.openqa.selenium.ElementNotInteractableException
(1 answer)
Closed 4 years ago.
Hello everyone forgive I think I can not really explain my problem, I will try again, if they can not understand me very well is that sometimes my English written fails a lot sorry
What I want to try is to automate the access to a web that I leave and link here
RUNT
The first part I have solved that is to enter the data to the form and resolve the im not to robot in send
I'm going to post all the code in python
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
import threading
import time
import csv
import os
# options = webdriver.ChromeOptions()
# options.add_argument(
# r'user-data-dir=C:\Users\RADEON\AppData\Local\Google\Chrome\user Data\default')
#
# options.add_extension(r"C:\Users\RADEON\Documents\Web Driver\Selenium\exs.crx")
# driver = webdriver.Chrome(
# executable_path="C:\\Users\\RADEON\\Documents\\Web Driver\\chrome Driver\\chromedriver.exe",
# chrome_options=options
# )
#
# driver = webdriver.Firefox()
#
# driver.get("https://www.runt.com.co/consultaCiudadana/#/consultaVehiculo")
# assert "Consulta Ciudadano - RUNT" in driver.title
#
# wait = WebDriverWait(driver, 2)
# wait.until(EC.presence_of_element_located((By.ID, "noPlaca")))
#
#
# wait.until(EC.presence_of_element_located((By.ID, "noPlaca")))
class Runt:
def __init__(self, placa, nit, time):
self.placa = placa
self.nit = nit
self.options = webdriver.ChromeOptions()
self.options.add_extension(
r"C:\Users\RADEON\Documents\Web Driver\cp.crx")
self.driver = webdriver.Chrome(
chrome_options=self.options)
self.wait = WebDriverWait(self.driver, time)
self.wait_API = WebDriverWait(self.driver, 150)
def closeBrowser(self):
self.driver.close()
def run(self):
driver = self.driver
wait = self.wait
wait_api = self.wait_API
driver.get("https://www.runt.com.co/consultaCiudadana/#/consultaVehiculo")
wait.until(EC.presence_of_element_located((By.ID, "noPlaca")))
placa = driver.find_element_by_id('noPlaca')
placa.clear()
placa.send_keys(self.placa)
wait.until(EC.presence_of_element_located((By.NAME, "noDocumento")))
owner = driver.find_element_by_name('noDocumento')
owner.clear()
owner.send_keys(self.nit)
# /html/body/div[1]/div/div[1]/div[2]/div[1]/div[2]/div[7]/div[1]
wait_api.until(EC.presence_of_element_located((
By.XPATH, "//*[#id='captcha']/div/div[2]/a[1]")))
while (driver.find_element_by_xpath("//*[#id='captcha']/div/div[2]/a[1]").get_attribute("innerText") != "Solved"):
print("Search Solution....")
print("solution found...")
if(driver.find_element_by_xpath("//*[#id='captcha']/div/div[2]/a[1]").get_attribute("innerText") == "Solved"):
driver.find_element_by_xpath(
"/html/body/div[1]/div/div[1]/div[2]/div[1]/div[2]/div[1]/div[3]/div[2]/div/div/form/div[8]/button").click()
WebDriverWait(driver, 20).until(EC.element_to_be_clickable(
(By.CSS_SELECTOR, "div.panel.panel-default>div.panel-heading>h4.panel-title a"))).click()
vigente = driver.find_element_by_xpath(
"//*[#id='pnlRevisionTecnicoMecanicaNacional']/div/div/div/table/tbody/tr[1]/td[5]")
print(vigente.get_attribute("innerText"))
runt2 = Runt("aqd470", 63364079, 2)
# runt1 = Runt("aqd470", 45884847, 2)
#
# thread1 = threading.Thread(target=runt1.run)
thread2 = threading.Thread(target=runt2.run)
#
# thread1.start()
thread2.start()
# r'C:\Users\RADEON\Documents\Web Driver\csv.csv'
Ignore the threads was doing some tests.
When running this program on the aforementioned web waiting to solve the I am not a robot and send the form, and the code appears that I want to get the information
but the information does not appear in the html until clicking on the following div
<div class="panel-heading" ng-click="togglePanel('pnlRevisionTecnicoMecanicaNacional');
consultarDetalle('RevisionTecnicoMecanicaNacional')">
<h4 class="panel-title">
<i class="i_hoja s_25_o1 ib"></i>
<a> Certificado de revisión técnico mécanica y de gases (RTM)</a>
</h4>
</div>
U can use this example data to enter "AFD329" For Nplaca,"6656954" For Documento
The other fields can be left as default
I need to click on that element to load the rest of the query would appreciate a lot if you can help me the TRACEBACK IS
Traceback (most recent call last):
File "C:\Users\RADEON\AppData\Local\Programs\Python\Python36-32\lib\threading.py", line 916, in _bootstrap_inner
self.run()
File "C:\Users\RADEON\AppData\Local\Programs\Python\Python36-32\lib\threading.py", line 864, in run
self._target(*self._args, **self._kwargs)
File "bypass.py", line 86, in run
"/html/body/div[1]/div/div[1]/div[2]/div[1]/div[2]/div[7]/div[1]").click()
File "C:\Users\RADEON\AppData\Local\Programs\Python\Python36-32\lib\site-packages\selenium\webdriver\remote\webelement.py", line 80, in click
self._execute(Command.CLICK_ELEMENT)
File "C:\Users\RADEON\AppData\Local\Programs\Python\Python36-32\lib\site-packages\selenium\webdriver\remote\webelement.py", line 633, in _execute
return self._parent.execute(command, params)
File "C:\Users\RADEON\AppData\Local\Programs\Python\Python36-32\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "C:\Users\RADEON\AppData\Local\Programs\Python\Python36-32\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.ElementNotVisibleException: Message: element not interactable
(Session info: chrome=70.0.3538.102)
(Driver info: chromedriver=2.43.600210 (68dcf5eebde37173d4027fa8635e332711d2874a),platform=Windows NT 10.0.16299 x86_64)