Develop Reference

Selenium having trouble finding elements

I am trying to gather recent NFT transaction activity from an OpenSea profile. I am getting an error that states the program is unable to locate the element, however my element has been defined and I copy and pasted the element from the web page.
What I am trying to do is selecting the class that holds all of the activities on the section, and then using XPath to gather each items info and print the item info. After looking at guides, videos, and other peoples code, I am pretty lost on what I am doing wrong.
Any advice would be greatly appreciated.
Here is my code:
from selenium import webdriver
from selenium.webdriver.common.by import By
PATH = ('/Users/chris/Desktop/chromedriver')
driver = webdriver.Chrome(PATH)
driver.get("https://opensea.io/GaryVee?tab=activity")
action = driver.find_element(By.CLASS_NAME, 'Blockreact__Block-sc-1xf18x6-0 dBFmez')
for action in actions:
nft_name = action.find_element(By.XPATH, './/*[#id="main"]/div/div/div[3]/div/div[3]/div[2]/div/div[2]/div['
'1]/button/div/div[2]/div/div/div/div[2]/span[1]/div/div/a').text
nft_id = action.find_element(By.XPATH, './/*[#id="main"]/div/div/div[3]/div/div[3]/div[2]/div/div[2]/div['
'1]/button/div/div[2]/div/div/div/div[2]/span[2]/a/div').text
price_eth = action.find_element(By.XPATH, './/*[#id="main"]/div/div/div[3]/div/div[3]/div[2]/div/div[2]/div['
'1]/button/div/div[3]/div/div[1]/div/div[2]/text()').text
price_usd = action.find_element(By.XPATH, './/*[#id="main"]/div/div/div[3]/div/div[3]/div[2]/div/div[2]/div['
'1]/button/div/div[3]/div/div[2]/span/div/div').text
sending_user = action.find_element(By.XPATH, './/*[#id="main"]/div/div/div[3]/div/div[3]/div[2]/div/div[2]/div['
'1]/button/div/div[5]/div/a/span').text
recieving_user = action.find_element(By.XPATH, './/*[#id="main"]/div/div/div[3]/div/div[3]/div[2]/div/div['
'2]/div[1]/button/div/div[6]/div/a/span').text
status = action.find_element(By.XPATH, './/*[#id="main"]/div/div/div[3]/div/div[3]/div[2]/div/div[2]/div['
'1]/button/div/div[1]/h6').text
print(nft_name, nft_id, price_eth, price_usd, sending_user, recieving_user, status)
Here is my error code:
/Users/chris/PycharmProjects/pythonProject/main.py:6: DeprecationWarning: executable_path has been deprecated, please pass in a Service object
driver = webdriver.Chrome(PATH)
Traceback (most recent call last):
File "/Users/chris/PycharmProjects/pythonProject/main.py", line 11, in <module>
action = driver.find_element(By.CLASS_NAME, 'Blockreact__Block-sc-1xf18x6-0 dBFmez')
File "/Users/chris/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/selenium/webdriver/remote/webdriver.py", line 1244, in find_element
return self.execute(Command.FIND_ELEMENT, {
File "/Users/chris/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/selenium/webdriver/remote/webdriver.py", line 424, in execute
self.error_handler.check_response(response)
File "/Users/chris/PycharmProjects/pythonProject/venv/lib/python3.9/site-packages/selenium/webdriver/remote/errorhandler.py", line 247, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"css selector","selector":".Blockreact__Block-sc-1xf18x6-0 dBFmez"}
(Session info: chrome=127.0.0.1)
Stacktrace:

To identify the element you can use either of the following Locator Strategies:
Using css_selector:
options = Options()
options.add_argument("start-maximized")
s = Service('C:\\BrowserDrivers\\chromedriver.exe')
driver = webdriver.Chrome(service=s, options=options)
driver.get("https://opensea.io/GaryVee?tab=activity")
action = driver.find_element(By.CSS_SELECTOR, ".Blockreact__Block-sc-1xf18x6-0.Flexreact__Flex-sc-1twd32i-0.FlexColumnreact__FlexColumn-sc-1wwz3hp-0.OpenSeaPagereact__DivContainer-sc-65pnmt-0.dBFmez.jYqxGr.ksFzlZ.iRiGb")
Using xpath:
options = Options()
options.add_argument("start-maximized")
s = Service('C:\\BrowserDrivers\\chromedriver.exe')
driver = webdriver.Chrome(service=s, options=options)
driver.get("https://opensea.io/GaryVee?tab=activity")
action = driver.find_element(By.XPATH, "//*[#class='Blockreact__Block-sc-1xf18x6-0 Flexreact__Flex-sc-1twd32i-0 FlexColumnreact__FlexColumn-sc-1wwz3hp-0 OpenSeaPagereact__DivContainer-sc-65pnmt-0 dBFmez jYqxGr ksFzlZ iRiGb']")

How to perform a search in LinkedIn with Selenium web driver

I am generating a bot to scrape LinkedIn profiles. At this time I am able to log into my account. The next step is to enter a name into the search bar.
I've written this script to get started:
# connect python with webbrowser-chrome
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.ui import WebDriverWait
import time
from selenium.webdriver.support.ui import WebDriverWait
import pyautogui as pag
def login_to_linkedin(driver):
username = driver.find_element_by_id("session_key")
username.send_keys("xxxxx#gmail.com")
password = driver.find_element_by_id("session_password")
password.send_keys("password")
driver.find_element_by_class_name("sign-in-form__submit-button").click()
def take_a_screenshot(driver):
loc_time = time.localtime()
time_string = time.strftime("%m/%d/%Y", loc_time)
driver.save_screenshot(time_string+"_screenshot.png")
def goto_network_page(driver,network_url):
driver.get(network_url)
def send_requests_to_users(driver):
WebDriverWait(driver, 60).until(EC.presence_of_element_located((By.CLASS_NAME, "class name of an element")))
driver.find_element_by_id("global-nav-typeahead")
driver.send_keys("name")
# javaScript = "window.scrollBy(0,4000);"
# driver.execute_script(javaScript)
# n = int(input("Number of requests: "))
# for i in range(0, n):
# pag.click(441, 666)
# print("Done !")
def main():
# url of LinkedIn
url = "http://linkedin.com/"
# path to browser web driver
driver = webdriver.Chrome(ChromeDriverManager().install())
driver.get(url)
login_to_linkedin(driver)
send_requests_to_users(driver)
take_a_screenshot(driver)
main()
Expected:
the keys for the search should be entered as expected.
Actual:
nothing happens in the search bar.
The new error that I am receiving is:
Traceback (most recent call last):
File "/Users/evangertis/development/mop/source/security-tools/container_scanning/selenium_test.py", line 48, in <module>
main()
File "/Users/evangertis/development/mop/source/security-tools/container_scanning/selenium_test.py", line 45, in main
send_requests_to_users(driver)
File "/Users/evangertis/development/mop/source/security-tools/container_scanning/selenium_test.py", line 33, in send_requests_to_users
driver.find_element(By.CSS_SELECTOR, ".search-global-typeahead button").click()
File "/usr/local/lib/python3.9/site-packages/selenium/webdriver/remote/webelement.py", line 80, in click
self._execute(Command.CLICK_ELEMENT)
File "/usr/local/lib/python3.9/site-packages/selenium/webdriver/remote/webelement.py", line 633, in _execute
return self._parent.execute(command, params)
File "/usr/local/lib/python3.9/site-packages/selenium/webdriver/remote/webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "/usr/local/lib/python3.9/site-packages/selenium/webdriver/remote/errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.ElementNotInteractableException: Message: element not interactable
(Session info: chrome=97.0.4692.71)

Try the following:
search_container = driver.find_element(By.CSS_SELECTOR, '.search-global-typeahead')
# Click on the search button if the input is not in focus
if 'focused' not in search_container.get_attribute('class'):
driver.find_element(By.CSS_SELECTOR, ".search-global-typeahead button").click()
driver.find_element(By.CSS_SELECTOR, "#global-nav-typeahead input").send_keys("name")

Try the following (based on Nic's answer):
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
search_text = 'Name'
search_container = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, '.search-global-typeahead')))
if 'focused' not in search_container.get_attribute('class'):
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, '.search-global-typeahead button'))).click()
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, '#global-nav-typeahead input'))).send_keys(search_text)
search_results = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, '.search-global-typeahead__hit-text')))
if 'focused' not in search_results.get_attribute('class'):
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, '.search-global-typeahead__hit-text'))).click()

How do i select a texbox and search a content and chose first option in selenium - python?

Hey I have this site : https://realmeupdater.com/
I want to search my query and get the first result url by clicking it. how can i do this? I tried to click element and send query but it gives me some error.
Here is The Code:
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.webdriver.chrome.options import Options
import asyncio
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver import ActionChains
GOOGLE_CHROME_BIN = 'path'
CHROME_DRIVER = 'path'
async def realme_rom_search(query):
url = "https://realmeupdater.com/"
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.binary_location = GOOGLE_CHROME_BIN
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--start-maximized")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-gpu")
driver = webdriver.Chrome(executable_path=CHROME_DRIVER, options=chrome_options)
driver.get(url)
await asyncio.sleep(5)
wait = WebDriverWait(driver, 20)
select_ccs = "select2-device-container"
wait.until(expected_conditions.element_to_be_clickable((By.CSS_SELECTOR, select_ccs))).click()
elem = wait.until(expected_conditions.visibility_of_element_located((By.CSS_SELECTOR, "body > span > span > span.select2-search.select2-search--dropdown > input")))
elem.send_keys(query)
elem.send_keys(Keys.DOWN)
elem.send_keys(Keys.ENTER)
But i Get This Error:
File "<string>", line 30, in realme_rom_search
File "/usr/local/lib/python3.9/site-packages/selenium/webdriver/remote/webelement.py", line 80, in click
self._execute(Command.CLICK_ELEMENT)
File "/usr/local/lib/python3.9/site-packages/selenium/webdriver/remote/webelement.py", line 633, in _execute
return self._parent.execute(command, params)
File "/usr/local/lib/python3.9/site-packages/selenium/webdriver/remote/webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "/usr/local/lib/python3.9/site-packages/selenium/webdriver/remote/errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.ElementClickInterceptedException: Message: element click intercepted: Element <span class="select2-selection__rendered" id="select2-device-container" role="textbox" aria-readonly="true">...</span> is not clickable at point (457, 586). Other element would receive the click: <div role="dialog" aria-live="polite" aria-label="cookieconsent" aria-describedby="cookieconsent:desc" class="cc-window cc-banner cc-type-opt-out cc-theme-classic cc-bottom cc-color-override--1142219405 " style="">...</div>
(Session info: headless chrome=90.0.4430.212)
All I want to do is search and go to the url and get rom info and url, can anyone help me?
Thank You in advance.

First you need to scroll till the end of the web page to let selenium know where is the drop down.
Use the below code to scroll till end :
driver.execute_script("var scrollingElement = (document.scrollingElement || document.body);scrollingElement.scrollTop = scrollingElement.scrollHeight;")
Secondly this drop is built using UI and LI elements so Select class from Selenium will not work. All you need to do is to store all elements in a list and then you can look for element which is desired and after string comparison you can click on it. Below is the sample code :
Code :
driver = webdriver.Chrome("C:\\Users:\\Desktop\\Selenium+Python\\chromedriver.exe")
driver.maximize_window()
wait = WebDriverWait(driver, 30)
driver.get("https://realmeupdater.com/")
sleep(2)
driver.execute_script("var scrollingElement = (document.scrollingElement || document.body);scrollingElement.scrollTop = scrollingElement.scrollHeight;")
wait.until(EC.visibility_of_element_located((By.ID, "select2-device-container"))).click()
all_options = driver.find_elements(By.CSS_SELECTOR, "#select2-device-results li")
for option in all_options:
if option.text =='realme 2 Pro (RMX1801EX)':
option.click()
print("Operation done successfully")

There are several issues here:
The locator you are using is not unique. This is not causing problems in this specific case, but generally it would be better to use unique locator.
The element you trying to click is initially out of the visible screen, so you first need to scroll it into the view.
There is a elements block asking to accept cookies on the bottom. That element is currently actually accepting your click. However I guess this could be resolved by scrolling your element into view.
So, please try the following:
from selenium.webdriver.common.action_chains import ActionChains
element = driver.find_element_by_css_selectorid(select_ccs)
actions = ActionChains(driver)
actions.move_to_element(element).perform()
This should help.
If it still not enough, try closing the cookies block before that so it will be:
from selenium.webdriver.common.action_chains import ActionChains
wait.until(expected_conditions.element_to_be_clickable((By.CSS_SELECTOR, 'a[aria-label="deny cookies"]'))).click()
wait.until(expected_conditions.element_to_be_clickable((By.CSS_SELECTOR, 'a[aria-label="dismiss cookie message"]'))).click()
element = driver.find_element_by_css_selectorid(select_ccs)
actions = ActionChains(driver)
actions.move_to_element(element).perform()

24/7 Python selenium "while true:" script can't execute driver.get("URL")

This is an odd problem
My script has a while True: script running 24/7. It always hits print ('False') every sleep(55) seconds, and once the current_time is between specified times, it calls driver.get("URL") and does a few actions on a website.
But sometimes it hits an error where it cannot open driver.get('https://www.instagram.com/accounts/login/?source=auth_switcher') and prints the error below.
Traceback (most recent call last):
File "/home/matt/insta/users/nycforest/lcf.py", line 254, in <module>
lcf_time(input_begin_time,input_end_time,input_begin_time2,input_end_time2)
File "/home/matt/insta/users/nycforest/lcf.py", line 241, in lcf_time
login()
File "/home/matt/insta/users/nycforest/lcf.py", line 40, in login
driver.get('https://www.instagram.com/accounts/login/?source=auth_switcher')
File "/home/matt/.local/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 333, in get
self.execute(Command.GET, {'url': url})
File "/home/matt/.local/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "/home/matt/.local/lib/python3.8/site-packages/selenium/webdriver/remote/errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message: timeout
(Session info: headless chrome=78.0.3904.70)
(Driver info: chromedriver=2.42.591071 (0b695ff80972cc1a65a5cd643186d2ae582cd4ac),platform=Linux 5.4.0-1020-aws x86_64)
I tried adding driver.set_page_load_timeout(20) after driver.get('URL') but it hits the error above
from pyvirtualdisplay import Display
from datetime import datetime, time
from itertools import islice
from random import randint
from time import sleep
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException
input_begin_time = time(18,40)
input_end_time = time(18,50)
input_begin_time2 = time(21,20)
input_end_time2 = time(21,30)
opts = webdriver.ChromeOptions()
opts.add_argument('--headless')
opts.add_argument('--disable-gpu')
opts.add_argument('--no-sandbox')
opts.add_argument('--disable-dev-shm-usage')
opts.add_argument('--enable-features=NetworkService,NetworkServiceInProcess')
def login():
sleep(2)
driver.get('https://www.instagram.com/accounts/login/?source=auth_switcher')
driver.set_page_load_timeout(20)
sleep(3)
input_username = driver.find_element_by_name('username').send_keys(username)
input_password = driver.find_element_by_name('password').send_keys(password)
button_login = driver.find_element_by_css_selector('#loginForm > div > div:nth-child(3) > button')
button_login.click()
# def other_functions()
driver = webdriver.Chrome(options=opts)
def lcf_time(time_begin1, time_end1, time_begin2, time_end2, curren_time=None):
current_time = datetime.now().time()
if time_begin1 < current_time < time_end1 or time_begin2 < current_time < time_end2:
login()
# other_functions()
driver.close()
else:
print("False")
while True:
lcf_time(input_begin_time,input_end_time,input_begin_time2,input_end_time2)
sleep(55)
The odd thing is: I have a few different scripts running the exact same script with different variables and it some scripts are fine and others consistently hit this 60% of the time. I need a more reliable way to driver.get("URL") 24/7
Is this a compatibility issue? Am I doing something wrong?

Instead of doing it with Selenium, you can do it with Python only or "curl" command.
Read here how to login to instagram using its exposed API for developers
In this way you can create a bash/shell script to trigger the script when you want and you won't need to keep the IDE running.

login into to a website using selenium python

I am trying to scrape some information from the below website. I am new to this so dont fully understand what is happening. I have basically been sourcing information from various examples i have found on internet
I am using selenium and python to help me navigate to the page from where i can scrape some information
I have used the below script
I am able to navigate to the home page , close cookies and click on sign in .
at this point a pop up opens up for entering user id and password
the div element is not being identified , each time i get error saying no such element is present
i added the wait with expected conditions , however i get the below error
C:\Users\user\AppData\Local\Programs\Python\Python38-32\SeleniumWebscraper2.py:14: DeprecationWarning: use options instead of chrome_options
browser = webdriver.Chrome(executable_path="C:/Users/user/Downloads/chromedriver_win32/chromedriver.exe", chrome_options=chrome_options)
DevTools listening on ws://127.0.0.1:2672/devtools/browser/f6aca700-5569-4367-9ceb-71e88fcc3082
CDwindow-BD7440D2580236DB9EEFE8B8FE1730D6
0
Traceback (most recent call last):
File "C:\Users\user\AppData\Local\Programs\Python\Python38-32\SeleniumWebscraper2.py", line 37, in
browser.find_element_by_xpath("//button[#class='gigya-input-text']").send_keys('x14127032#student.ncirl.ie')
File "C:\Users\user\AppData\Local\Programs\Python\Python38-32\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 394, in find_element_by_xpath
return self.find_element(by=By.XPATH, value=xpath)
File "C:\Users\user\AppData\Local\Programs\Python\Python38-32\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 976, in find_element
return self.execute(Command.FIND_ELEMENT, {
File "C:\Users\user\AppData\Local\Programs\Python\Python38-32\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "C:\Users\user\AppData\Local\Programs\Python\Python38-32\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"xpath","selector":"//button[#class='gigya-input-text']"}
(Session info: chrome=78.0.3904.97)
https://education.independent.ie/league
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
import requests
from bs4 import BeautifulSoup
chrome_options = webdriver.ChromeOptions()
chrome_options.add_experimental_option("prefs", {"profile.default_content_settings.cookies": 2})
browser = webdriver.Chrome(executable_path="C:/Users/user/Downloads/chromedriver_win32/chromedriver.exe", chrome_options=chrome_options)
browser.get("https://education.independent.ie/league/school/abbey-community-college-roscommon-558")
browser.implicitly_wait(10) # seconds
selector = ".gigya-input-text"
main_window_handle = None
while not main_window_handle:
main_window_handle = browser.current_window_handle
print(main_window_handle)
browser.find_element_by_xpath("//button[#class='qc-cmp-button']").click()
WebDriverWait(browser, 10)
browser.switch_to.window(browser.current_window_handle)
browser.find_element_by_xpath("//button[#class='button-b gigya-sign-in']").send_keys('\n')
WebDriverWait(browser, 120).until(EC.visibility_of_element_located((By.CSS_SELECTOR, selector )))
print(len(browser.find_elements_by_id("gigya-input-text")))
browser.find_element_by_xpath("//button[#class='gigya-input-text']").send_keys('x14127032#student.ncirl.ie')

try this:
# import web driver
from selenium import webdriver
# specifies the path to the chromedriver.exe
driver = webdriver.Chrome("/opt/anaconda2/chromedriver")
# driver.get method() will navigate to a page given by the URL address
driver.get("https://www.linkedin.com/login?")
# locate email form by_name
username = driver.find_element_by_name('session_key')
# locate email form by_xpath
#username = driver.find_element_by_xpath('//*[#id="username"]')
# locate email form by_id
#username = driver.find_element_by_id("username")
#send_keys() to simulate key strokes
username.send_keys("username#email.com")
# locate password form by_name
#password = driver.find_element_by_name('session_password')
#locate password form id
#password = driver.find_element_by_id("password")
#locate password form by_xpath
password = driver.find_element_by_xpath('//*[#id="password"]')
# send_keys() to simulate key strokes
password.send_keys("password")
# locate submit button by_class_name
log_in_button = driver.find_element_by_class_name('btn__primary--large')
# locate submit button by_xpath
#log_in_button = driver.find_element_by_xpath('//*[#type="submit"]')
# .click() to mimic button click
log_in_button.click()

You are getting this error because the part of id gig_1574604202288_showScreenSet that your are trying to find is random. You can use other selector to locate this element e.g. (By.CSS_SELECTOR, "div[id*='showScreenSet']"). You can read about this more here and practice here.
Also WebDriverWait(browser, 120) does nothing. If you want global wait you can use implicit wait. Read about this here.