Develop Reference

How to use the dropwon selenium selector?

I'm trying to use the selenium selector but I'm still getting this error message.
When I try 3Select(web.find_element("id",'ticket-quantity-selector-756706789'))", it doesn't seem to work on my terminal.
Thank you
self.execute(Command.GET, {'url': url})
File "/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages/selenium/webdriver/remote/webdriver.py", line 429, in execute
self.error_handler.check_response(response)
File "/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/site-packages/selenium/webdriver/remote/errorhandler.py", line 243, in check_response
-------------------------
from selenium import webdriver
from selenium.webdriver.support.ui import Select
import time
web = webdriver.Chrome()
web.get('https://www.eventbrite.fr/e/masquerade-party-tickets-443305678217')
time.sleep(5)
cookie = web.find_element("xpath",'//*[#id="_evidon-accept-button"]')
cookie.click()
info = web.find_element("xpath",'//*[#id="eventbrite-widget-modal-trigger-443305678217"]')
info.click()
CoupC = Select(web.find_element("id",'ticket-quantity-selector-756706789'))
CoupC.select_by_visible_text("5")
//*[#id="ticket-quantity-selector-756706789"]
CoupS = Select(web.find_element("id",'ticket-quantity-selector-756706799'))
CoupS.select_by_visible_text("5")
Cliquer = web.find_element("xpath",'//*[#id="root"]/div/div/div/div/div[1]/div[1]/div/main/div/div[2]/div/div/div[2]/button')
Cliquer.click()
time.sleep(3)

Dropdowns are inside an iframe, so first you have to switch to iframe, then you have to select the option, try this, it's working:
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
WebDriverWait(driver, 10).until(EC.frame_to_be_available_and_switch_to_it(driver.find_element(By.ID, "eventbrite-widget-modal-443305678217")))
time.sleep(2)
driver.find_element(By.ID,'ticket-quantity-selector-756706789').click()
CoupC = Select(driver.find_element(By.ID,'ticket-quantity-selector-756706789'))
CoupC.select_by_visible_text("5")
CoupS = Select(driver.find_element(By.ID,'ticket-quantity-selector-756706799'))
CoupS.select_by_visible_text("5")
Cliquer = driver.find_element(By.XPATH,'//*[#id="root"]/div/div/div/div/div[1]/div[1]/div/main/div/div[2]/div/div/div[2]/button')
Cliquer.click()
If you need to switch to the main page from iframe, add this:
driver.switch_to.default_content()

How to perform a search in LinkedIn with Selenium web driver

I am generating a bot to scrape LinkedIn profiles. At this time I am able to log into my account. The next step is to enter a name into the search bar.
I've written this script to get started:
# connect python with webbrowser-chrome
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.ui import WebDriverWait
import time
from selenium.webdriver.support.ui import WebDriverWait
import pyautogui as pag
def login_to_linkedin(driver):
username = driver.find_element_by_id("session_key")
username.send_keys("xxxxx#gmail.com")
password = driver.find_element_by_id("session_password")
password.send_keys("password")
driver.find_element_by_class_name("sign-in-form__submit-button").click()
def take_a_screenshot(driver):
loc_time = time.localtime()
time_string = time.strftime("%m/%d/%Y", loc_time)
driver.save_screenshot(time_string+"_screenshot.png")
def goto_network_page(driver,network_url):
driver.get(network_url)
def send_requests_to_users(driver):
WebDriverWait(driver, 60).until(EC.presence_of_element_located((By.CLASS_NAME, "class name of an element")))
driver.find_element_by_id("global-nav-typeahead")
driver.send_keys("name")
# javaScript = "window.scrollBy(0,4000);"
# driver.execute_script(javaScript)
# n = int(input("Number of requests: "))
# for i in range(0, n):
# pag.click(441, 666)
# print("Done !")
def main():
# url of LinkedIn
url = "http://linkedin.com/"
# path to browser web driver
driver = webdriver.Chrome(ChromeDriverManager().install())
driver.get(url)
login_to_linkedin(driver)
send_requests_to_users(driver)
take_a_screenshot(driver)
main()
Expected:
the keys for the search should be entered as expected.
Actual:
nothing happens in the search bar.
The new error that I am receiving is:
Traceback (most recent call last):
File "/Users/evangertis/development/mop/source/security-tools/container_scanning/selenium_test.py", line 48, in <module>
main()
File "/Users/evangertis/development/mop/source/security-tools/container_scanning/selenium_test.py", line 45, in main
send_requests_to_users(driver)
File "/Users/evangertis/development/mop/source/security-tools/container_scanning/selenium_test.py", line 33, in send_requests_to_users
driver.find_element(By.CSS_SELECTOR, ".search-global-typeahead button").click()
File "/usr/local/lib/python3.9/site-packages/selenium/webdriver/remote/webelement.py", line 80, in click
self._execute(Command.CLICK_ELEMENT)
File "/usr/local/lib/python3.9/site-packages/selenium/webdriver/remote/webelement.py", line 633, in _execute
return self._parent.execute(command, params)
File "/usr/local/lib/python3.9/site-packages/selenium/webdriver/remote/webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "/usr/local/lib/python3.9/site-packages/selenium/webdriver/remote/errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.ElementNotInteractableException: Message: element not interactable
(Session info: chrome=97.0.4692.71)

Try the following:
search_container = driver.find_element(By.CSS_SELECTOR, '.search-global-typeahead')
# Click on the search button if the input is not in focus
if 'focused' not in search_container.get_attribute('class'):
driver.find_element(By.CSS_SELECTOR, ".search-global-typeahead button").click()
driver.find_element(By.CSS_SELECTOR, "#global-nav-typeahead input").send_keys("name")

Try the following (based on Nic's answer):
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
search_text = 'Name'
search_container = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, '.search-global-typeahead')))
if 'focused' not in search_container.get_attribute('class'):
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, '.search-global-typeahead button'))).click()
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, '#global-nav-typeahead input'))).send_keys(search_text)
search_results = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, '.search-global-typeahead__hit-text')))
if 'focused' not in search_results.get_attribute('class'):
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, '.search-global-typeahead__hit-text'))).click()

Selenium Python - Stale Element Reference Exception

I'm trying to extract some public information from a government site and need to scroll through the pages by clicking "Próximo", but I'm getting the following error
Traceback (most recent call last):
File "/home/lantri_rafael/codigo/diariosbr/raf/dom-saojoaquimdabarra/coleta_selenium3.py", line 39, in <module>
main()
File "/home/lantri_rafael/codigo/diariosbr/raf/dom-saojoaquimdabarra/coleta_selenium3.py", line 36, in main
pag = page()
File "/home/lantri_rafael/codigo/diariosbr/raf/dom-saojoaquimdabarra/coleta_selenium3.py", line 19, in page
pag.click()
File "/opt/anaconda/envs/env_diariosbr/lib/python3.9/site-packages/selenium/webdriver/remote/webelement.py", line 80, in click
self._execute(Command.CLICK_ELEMENT)
File "/opt/anaconda/envs/env_diariosbr/lib/python3.9/site-packages/selenium/webdriver/remote/webelement.py", line 633, in _execute
return self._parent.execute(command, params)
File "/opt/anaconda/envs/env_diariosbr/lib/python3.9/site-packages/selenium/webdriver/remote/webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "/opt/anaconda/envs/env_diariosbr/lib/python3.9/site-packages/selenium/webdriver/remote/errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.StaleElementReferenceException: Message: The element reference of [object String] "{\"element-6066-11e4-a52e-4f735466cecf\":\"d099f837-9079-4668-a534-3099fc73a76c\"}" is stale; either the element is no longer attached to the DOM, it is not in the current frame context, or the document has been refreshed
Below the code and site
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from time import sleep
def page():
pag = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, '//*[#id="Pagination"]/a[#class="next"]')))
list_ed = driver.find_elements_by_xpath('//*[#id="jornal"]/li[#class="lista"]')
def info():
for ed in list_ed:
el_text = ed.text
el = ed.find_element_by_xpath('a').get_attribute('href')
print(el_text)
print(el)
info()
pag.click()
sleep(35)
info()
def main():
global driver
driver = webdriver.Firefox()
driver.get("https://www.imprensaoficialmunicipal.com.br/sao_joaquim_da_barra")
driver.implicitly_wait(10)
pag = page()
if __name__ == '__main__':
main()
any suggestions on how to resolve this?

Problem is because click() loads new HTML in browser and you have to use again find_elements_by_xpath to get elements in new HTML. But you use all time the same list with references to elements on first page and it can't find these elements on next pages.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from time import sleep
def info(driver, delay=5):
next_page = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, '//*[#id="Pagination"]/a[#class="next"]')))
all_elements = driver.find_elements_by_xpath('//*[#id="jornal"]/li[#class="lista"]')
for item in all_elements:
text = item.text
href = item.find_element_by_xpath('a').get_attribute('href')
print(text)
print('href:', href)
print('---')
next_page.click()
sleep(delay)
def page(driver):
for _ in range(5):
#info(driver)
info(driver, 2)
def main():
driver = webdriver.Firefox()
driver.get("https://www.imprensaoficialmunicipal.com.br/sao_joaquim_da_barra")
driver.implicitly_wait(10)
pag = page(driver)
if __name__ == '__main__':
main()

Selenium stale element reference in for loop

I Used Windows10, Chrome version 89.0.4389.9, vscode, python
This code only loops once then errors with the error below.
table = driver.find_element_by_xpath('//*[#id="frm"]/table')
tbody = table.find_element_by_tag_name("tbody")
rows = tbody.find_elements_by_tag_name("tr")
# btns = driver.find_element_by_xpath('//*[#id="frm"]/table/tbody/tr[*]/td[2]/a')
for index, value in enumerate(rows):
body=value.find_elements_by_tag_name("td")[1]
body.click()
sleep(2)
driver.back()
sleep(2)
Traceback (most recent call last):
File "d:/Study/Companylist/program/pandastest.py", line 80, in <module>
body=value.find_elements_by_tag_name("td")[1]
File "D:\Anaconda\lib\site-packages\selenium\webdriver\remote\webelement.py", line 320, in find_elements_by_tag_name
return self.find_elements(by=By.TAG_NAME, value=name)
File "D:\Anaconda\lib\site-packages\selenium\webdriver\remote\webelement.py", line 684, in find_elements
return self._execute(Command.FIND_CHILD_ELEMENTS,
File "D:\Anaconda\lib\site-packages\selenium\webdriver\remote\webelement.py", line 633, in _execute
return self._parent.execute(command, params)
File "D:\Anaconda\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "D:\Anaconda\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.StaleElementReferenceException: Message: stale element reference: element is not attached to the page document

Seems like I found the solution, but can protect it that it's the best, will dig into it more:
elements = driver.find_elements(By.CSS_SELECTOR, 'div.g')
for n, el in enumerate(elements):
elements = driver.find_elements(By.CSS_SELECTOR, 'div.g')
elements[n].click()
time.sleep(1)
driver.back()
time.sleep(1)
driver.quit()
try to find elements, then move start the loop and find the same result and get items from that loop by item number from enumarate function.
from time import sleep
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
def company_info(driver):
com_name = driver.find_element_by_xpath('Enter your site xpath')
print(com_name.text)
com_addr = driver.find_element_by_xpath('Enter your site xpath')
print(com_addr.text)
com_tel = driver.find_element_by_xpath('Enter your site xpath')
print(com_tel.text)
com_fax = driver.find_element_by_xpath('Enter your site xpath')
print(com_fax.text)
driver = webdriver.Chrome()
url_search = 'Enter your site URL'
#input values
web_open_wait = 5
web_close_wait = 3
driver.get(url_search)
sleep(web_open_wait)
check_names = driver.find_elements_by_xpath('//*[#id="frm"]/table/tbody/tr[1]/td/a'
for n, el in enumarate(check_names, start=1):
check_names = driver.find_elements_by_xpath('//*[#id="frm"]/table/tbody/tr[%d]/td/a' % n)
check_name[el].click()
company_info(driver)
driver.back()
driver.quit()

I don't use selenium, but I believe the problem is that the element is no longer in the DOM. To get around this, you could use a "try" block.
try:
body=value.find_elements_by_tag_name("td")[1]
#some code
except:
pass

We will find the answer. Like we always do
I was thinking of the code I just put on top, and I came up with this method when I was thinking about how to fix #Vovo, one of the hints you gave me that rows couldn't come in repeatedly.
It doesn't look pretty, but it works well.
(It is difficult to use two drivers because the site I open cannot click the control.)
Anyway. Share. Hope it helps if someone sees this.
What I was trying to make is,
Enter the site containing the company information,
Among the publicly available information,
I was trying to extract the company name, address, contact information, and fax number.
The order of operation is:
Site access> Set xpath path of desired data> Click> Extract> End site
It's repeating this cycle
As much as i want
For reference, the sites I open are like Control + Cilck or something like that. If you click it, you have to load the element unconditionally. Go back or go forward, so it keeps hurting.
It would be comfortable to open it up and extract it and turn it off and extract it and turn it off.
If there is any improvement, please tell me
from time import sleep
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
def company_info():
com_name = driver.find_element_by_xpath('Enter your site xpath')
print(com_name.text)
com_addr = driver.find_element_by_xpath('Enter your site xpath')
print(com_addr.text)
com_tel = driver.find_element_by_xpath('Enter your site xpath')
print(com_tel.text)
com_fax = driver.find_element_by_xpath('Enter your site xpath')
print(com_fax.text)
num_add = 0
while True:
driver = webdriver.Chrome()
url_search = 'Enter your site URL'
#input values
web_open_wait = 5
web_close_wait = 3
# Url Open
driver.get(url_search)
sleep(web_open_wait)
# Collect info
num_add += 1
check_name = driver.find_element_by_xpath('//*[#id="frm"]/table/tbody/tr[{0}]/td[2]/a'.format(num_add))
check_name.click()
sleep(web_open_wait)
company_info()
# Url Close
driver.quit()

Can't perform a click on a search button because of some ad hiding it

I've written a script in python with selenium to perform a search in a webpage using search-box available there. However, When I run my script, It throws an error which I'm gonna paste below. The thing is when the webpage is loaded through my script, there is an advertisement pops up hiding the search-box. How can I get around that and fetch the search result? Thanks in advance.
Link to that site: webpage
Script I'm trying with:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
driver = webdriver.Chrome()
wait = WebDriverWait(driver, 10)
driver.get("replace_with_above_site")
wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "input#q"))).send_keys("Camden Medical Centre, 1 Orchard Boulevard 248649")
wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "input#search_button"))).click() ##error thrown here
driver.quit()
Traceback I'm having:
Traceback (most recent call last):
File "C:\Users\WCS\AppData\Local\Programs\Python\Python36-32\demo.py", line 12, in <module>
wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "input#search_button"))).click()
File "C:\Users\WCS\AppData\Local\Programs\Python\Python36-32\lib\site-packages\selenium\webdriver\remote\webelement.py", line 80, in click
self._execute(Command.CLICK_ELEMENT)
File "C:\Users\WCS\AppData\Local\Programs\Python\Python36-32\lib\site-packages\selenium\webdriver\remote\webelement.py", line 628, in _execute
return self._parent.execute(command, params)
File "C:\Users\WCS\AppData\Local\Programs\Python\Python36-32\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 312, in execute
self.error_handler.check_response(response)
File "C:\Users\WCS\AppData\Local\Programs\Python\Python36-32\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.WebDriverException: Message: unknown error: Element <input type="button" id="search_button" onclick="submitSearch();"> is not clickable at point (868, 137). Other element would receive the click: <div id="splash_screen_overlay"></div>
This is the ad which hides the search box:
Btw, the search parameter is available within .send_keys() in my script. Anything from the populated result will suffice.

The simplest solution is to simulate exactly the same action user should do: close an ad in case it appeared:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
driver = webdriver.Chrome()
wait = WebDriverWait(driver, 10)
driver.get("replace_with_above_site")
try:
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "a.btn_close"))).click()
except NoSuchElementException:
pass
wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "input#q"))).send_keys("Camden Medical Centre, 1 Orchard Boulevard 248649")
wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "input#search_button"))).click()
driver.quit()

Use below code. This will click the element even if it is hidden beneath ad
((IJavaScriptExecutor)driver).ExecuteScript("arguments[0].scrollIntoView(true)", driver.GetElement(By.Id("someID")));