Develop Reference

How to click 'View' button using Selenium on Wunderground Historical Weather page

I'm trying to access the "View" button on the "https://www.wunderground.com/history" website using Selenium in Python but I'm unable to do so. I've tried multiple methods including:
ActionChains("actions.move_to_element(button_view).click().perform()")
browser.find_element(By.ID, 'dateSubmit').click()
wait.until(EC.element_to_be_clickable((By.ID, 'dateSubmit')))
actions.move_to_element(button_view).double_click(button_view)
but none of them seems to work.
Interestingly, when I tried to access a similar location page (https://www.wunderground.com/history/daily/KLGA/), all I needed to do was type a search in the 'historySearch' box and the button would click, but on the "/history" page, it wouldn't work no matter what I did.
I suspect the reason for this is due to the different actions performed before clicking the button, such as clicking the search text box area, entering a location (e.g. "New York, NY"), and selecting an option from the auto-completion dropdown menu.
In my case, I couldn't select the desired option from the dropdown menu, so I tried to unfocus from the text box, which in the site's case automatically chooses the first auto-completion option for you.
from selenium import webdriver
from selenium.webdriver.support.ui import Select
from datetime import date, timedelta
from selenium.webdriver.common.by import By
from selenium.webdriver import ActionChains
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.keys import Keys
from collections import Counter
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from bs4 import BeautifulSoup
from datetime import datetime
import time
import logging
import pandas as pd
def get_weather_data4( year, month, day, browser):
try:
# Open browser and navigate to the website
#browser = webdriver.Chrome()
wait = WebDriverWait(browser, 20)
browser.get('https://www.wunderground.com/history')
wait.until(EC.presence_of_element_located((By.ID, 'yearSelection')))
wait.until(EC.presence_of_element_located((By.ID, 'historySearch')))
# Select year, month, and day from the drop-down menus
actions = ActionChains(browser)
year_select = Select(browser.find_element(By.ID,'yearSelection'))
year_select.select_by_value(str((2023-year))+':'+' '+str(year))
month_select = Select(browser.find_element(By.ID,'monthSelection'))
month_select.select_by_value(str(month))
day_select = Select(browser.find_element(By.ID,'daySelection'))
day_select.select_by_value(str((day-1))+':'+' '+str(day))
search_field = wait.until(EC.visibility_of_element_located((By.ID, 'historySearch')))
search_field.click()
search_field.send_keys("New York, NY", Keys.ENTER)
#search_field.send_keys(Keys.TAB * 2 )#+ Keys.RETURN * 2)
try:
wait.until(EC.element_to_be_clickable((By.ID, 'dateSubmit')))
button_view = browser.find_element(By.ID, 'dateSubmit')
#button_view.send_keys("",Keys.ENTER)
actions.move_to_element(button_view).double_click(button_view)
except TimeoutError as te:
print(f"The following element caused a timeout error: {te}")
# Click the "View" button to load the data for the selected date FAILED ATTEMPTS
#button_view = wait.until(EC.element_to_be_clickable((By.ID, 'dateSubmit')))
#actions.move_to_element(button_view).click().perform()
#button_view.click()
#browser.find_element(By.ID, 'dateSubmit').click()
#WebDriverWait(browser, 2)
#browser.find_element(By.ID, 'dateSubmit').click()
# Wait for the data to load
wait.until_not(EC.presence_of_element_located((By.XPATH, '//div[#class="loading-indicator"]')))
wait.until(EC.visibility_of_element_located((By.CLASS_NAME, 'row')))
wait.until(EC.presence_of_element_located((By.XPATH, '//table[#class="mat-table cdk-table mat-sort ng-star-inserted"]')))
except TimeoutError:
raise Exception("Timeout Error: Could not load data from URL")
# Extract the data from the table using BeautifulSoup
soup = BeautifulSoup(browser.page_source, 'html.parser')
table = soup.select_one('table[class*="mat-table cdk-table mat-sort ng-star-inserted"]')
rows = table.find_all('tr',{'class':'mat-row cdk-row ng-star-inserted'})
# Code continues...
Relevant Details:
Selenium version: 4.7.2
Browser: Firefox
Operating System: Windows 10
Request: I am seeking assistance in resolving this issue and clicking the "View" button on the website. Any help or suggestions would be greatly appreciated.

To fill up the Location box with Boston, click on the first autocomplete item and click on the element View you need to induce WebDriverWait for the element_to_be_clickable() and you can use the following locator strategies:
driver.get('https://www.wunderground.com/history')
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "input#historySearch[name='historySearch']"))).send_keys("Boston")
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//search-autocomplete//ul[#class='ui-autocomplete ui-front ui-menu ui-widget ui-widget-content ui-corner-all']//li/a/span[contains(., 'MA')]/span[text()='Boston']"))).click()
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "input#dateSubmit[value='View']"))).click()
Note: You have to add the following imports :
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
Browser Snapshot:

Dropdown option not getting selected using Selenium Python

I am trying to select a dropdown option in the form using Selenium webdriver in Python. The XPATH is correct, I also verified it is going to the right dropdown option but in the end it is not selecting it.
I have tried similar code for another website that has a dropdown. But it's not working for this particular website.
Can someone please help out with this?
from selenium import webdriver
driver = webdriver.Chrome("C:\\Users\\xxx\\Downloads\\chromedriver_win32\\chromedriver.exe")
driver.get("https://www.cersai.org.in/CERSAI/dbtrsrch.prg")
elem = driver.find_element_by_xpath("//select[#id='borrowerType']")
all_options = elem.find_elements_by_tag_name("option")
for option in all_options:
if option.get_attribute("value") == "IND":
option.click()
break

You should add a wait before accessing the dropdown element to make it loaded.
Also, this is a Select element, you can treat it in a special way as below:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Chrome("C:\\Users\\xxx\\Downloads\\chromedriver_win32\\chromedriver.exe")
wait = WebDriverWait(driver, 20)
driver.get("https://www.cersai.org.in/CERSAI/dbtrsrch.prg")
wait.until(EC.visibility_of_element_located((By.XPATH, "//select[#id='borrowerType']")))
select = Select(driver.find_element_by_xpath("//select[#id='borrowerType']"))
# select by visible text
select.select_by_value('IND')

It's strange that Select class did not work. It needs a JavaScript call.
driver = webdriver.Chrome(driver_path)
driver.maximize_window()
driver.implicitly_wait(50)
driver.get("https://www.cersai.org.in/CERSAI/dbtrsrch.prg")
driver.execute_script("return document.getElementById('borrowerType').selectedIndex = '2'")

python selenium - Cant find the element of sign in button

I'm trying to create an automation test in Asos (for practice purpose only) however I'm having a hard time locating this sign-in element...
I need to click on that sign-in button.
these are the element I got in inspect:
a class="_1336dMe _1uUU2Co _1336dMe _1uUU2Co" href="https://my.asos.com/my-account?
lang=en-GB&store=COM&country=GB&keyStoreDataversion=3pmn72e-27"
data-testid="signin-link" tabindex="-1">Sign In

I had the same problem when trying to find this button on Google Maps. Is the sign in button on a pop up window? Then the problem is becuse you have to change between frames.
Here is a code sample:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Chrome('driver path')
url = 'url'
driver.get(url)
WebDriverWait(driver, 10).until(EC.frame_to_be_available_and_switch_to_it((By.XPATH, '//*[#id="consent-bump"]/div/div[1]/iframe')))
agree = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//*[#id="introAgreeButton"]/span/span')))
agree.click()
#back to the main page
driver.switch_to_default_content()
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//*[#id="searchboxinput"]'))).send_keys('gostilne')
search = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//*[#id="searchbox-searchbutton"]')))
search.click()
Make shure that the xpath for frames and buttons is correct.

Python (Selenium) Selecting a drop down list from HHPRED

I'm attempting to select drop down menu items from the HHPRED website. (URL: https://toolkit.tuebingen.mpg.de/tools/hhpred) and I keep bumping into either 'object not found,' or 'object not clickable/selectable'.
# Input protein from txt file (predator_file variable)
text_area = driver.find_element_by_id('__BVID__121')
text_area.send_keys(predator_file)
# Input PDB, SCOP, PFAM, and NCBI domains
first_click = driver.find_element_by_id('__BVID__130')
scop_click = driver.find_element_by_link_text("SCOPe")
pfam_click = driver.find_element_by_link_text("Pfam")
ncbi_click = driver.find_element_by_link_text("ncbi_")
I know I'm working on selenium correctly because the first portion for my text entry is copying correctly but, when I'm working on the drop down from selecting it to even picking what I need - I'm lost. See below the inspected elements for HHPRED and the drop down I'm looking at tackling.
Any help would be greatly appreciated!

Currently your url is unreachable due tooc credntials. You can use below code to select value/visible text from the dropdown.
from selenium import webdriver
from selenium.webdriver.support.ui import Select
select= WebDriverWait(driver, 30).until(
EC.element_to_be_clickable((By.XPATH, "select element xpath")))
print(len(select.options))
select.select_by_value("") # select by value
select.select_by_visible_text('') # select by visible text
Note : please add below imports to your solution
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
or
driver.find_element_by_xpath("//select[#name='element_name']/option[text()='option_text']").click()
Updated solution : its a custom dropdown element so you need to handle it in a differnt way . Kindly find below code for your reference. I have verified it and its working as expected .
driver.get("https://toolkit.tuebingen.mpg.de/tools/hhpred")
main_window = driver.current_window_handle
wait = WebDriverWait(driver, 20)
wait.until(EC.element_to_be_clickable((By.XPATH, "//button[#class='btn sign-in-link btn-href btn-sm']"))).click()
wait.until(EC.element_to_be_clickable((By.XPATH, "(//fieldset[#class='form-group']//input)[2]"))).send_keys('')
wait.until(EC.element_to_be_clickable((By.XPATH, "(//fieldset[#class='form-group']//input)[3]"))).send_keys('')
wait.until(EC.element_to_be_clickable((By.XPATH, "//button[#class='btn btn-secondary']"))).click()
wait.until(EC.element_to_be_clickable((By.XPATH, "//*[contains(text(), 'Got it!')]"))).click()
print wait.until(EC.element_to_be_clickable((By.XPATH, "//span[contains(text(),'Paste Example')]"))).text
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
clickElement=wait.until(EC.element_to_be_clickable((By.XPATH, "//div[#class='multiselect__tags']")))
ActionChains(driver).move_to_element(clickElement).click().perform()
wait.until(EC.element_to_be_clickable((By.XPATH, "//li[*]//span[contains(text(),'TIGRFAMs_v15.0')]"))).click()
Output:

Selenium crashes when I'm trying to parse the next page (and seven after it) on a website. Any way to tackle this?

I want to parse an IMDb film rating located here on around 8 pages. In order to do that I'm using Selenium, and I'm having trouble with clicks, proceeding algorithm to next page. In the end I need 1000 titles when I'll continue using BeautifulSoup. Code below isn't working, I need to use button 'NEXT' with this HTML:
<a class="flat-button lister-page-next next-page" href="/list/ls000004717/?page=2">
Next
</a>
This is the code:
from selenium import webdriver as wb
browser = wb.Chrome()
browser.get('https://www.imdb.com/list/ls000004717/')
field = browser.find_element_by_name("flat-button lister-page-next next-page").click()
Error is the following:
NoSuchElementException: Message: no such element: Unable to locate element: {"method":"css selector","selector":".flat-button lister-page-next next-page"}
(Session info: chrome=78.0.3904.108)
I suppose I lack knowledge of syntax needed, or maybe I mixed it up a little. I tried searching on SO, though every example is pretty unique and I don't possess the knowledge to extrapolate these cases fully. Any way Selenium can handle that?

You could try using an XPath to query on the Next text inside the button. You should also probably invoke WebDriverWait since you are navigating across multiple pages, then scroll into view since this is at the bottom of the page:
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from time import sleep
browser = wb.Chrome()
browser.get('https://www.imdb.com/list/ls000004717/')
# keep clicking next until we reach the end
for i in range(0,9):
# wait up to 10s before locating next button
try:
next_button = WebDriverWait(browser, 10).until(EC.element_to_be_clickable((By.XPATH, "//a[contains(#class, 'page') and contains(text(), 'Next')]")))
# scroll down to button using Javascript
browser.execute_script("arguments[0].scrollIntoView(true);", next_button)
# click the button
# next_button.click() this throws exception -- replace with JS click
browser.execute_script("arguments[0].click();", next_button)
# I never recommend using sleep like this, but WebDriverWait is not waiting on next button to fully load, so it goes stale.
sleep(5)
# case: next button no longer exists, we have reached the end
except TimeoutException:
break
I also wrapped everything in a try / except TimeoutException block to handle the case where we have reached the end of pages, and Next button no longer exists, thus breaking out of the loop. This worked on multiple pages for me.
I also had to add an explicit sleep(5) because even after invoking WebDriverWait on element_to_be_clickable, next_button was still throwing StaleElementReferenceException. It seems like WebDriverWait was finishing before page was fully loaded, causing the status of next_button to change after it had been located. Normally adding sleep(5) is bad practice, but there did not seem to be another workaround here. If anyone else has a suggestion on this, feel free to comment / edit the answer.

There are a couple of ways that could work:
1. Use a selector for the next button and loop until the end:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as ec
browser = webdriver.Chrome()
browser.get('https://www.imdb.com/list/ls000004717/')
selector = 'a[class*="next-page"]'
num_pages = 10
for page in range(pages):
# Wait for the element to load
WebDriverWait(browser, 10).until(ec.presence_of_element_located((By.CSS_SELECTOR, selector)))
# ... Do rating parsing here
browser.find_element_by_css_selector(selector).click()
Instead of clicking on the element, the other option could be to navigate to the next page using broswer.get('...'):
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as ec
# Set up browser as before and navigate to the page
browser = webdriver.Chrome()
browser.get('https://www.imdb.com/list/ls000004717/')
selector = 'a[class*="next-page"]'
base_url = 'https://www.imdb.com/list/ls000004717/'
page_extension = '?page='
# Already at page = 1, so only needs to loop 9 times
for page in range(2, pages + 1):
# Wait for the page to load
WebDriverWait(browser, 10).until(ec.presence_of_element_located((By.CSS_SELECTOR, selector)))
# ... Do rating parsing here
next_page = base_url + page_extension + str(page)
browser.get(next_page)
As a note: field = browser.find_element_by_name("...").click() will not assign field to a webelement, as the click() method has no return value.

You could try a partial css selector.
browser.find_element_by_css_selector("a[class*='next-page']").click()

To click on the element with text as NEXT till the 901 - 1,000 of 1,000 page you have to:
scrollIntoView() the element once the visibility_of_element_located() is achieved.
Induce WebDriverWait for the element_to_be_clickable()
You can use the following solution:
Code Block:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
options = webdriver.ChromeOptions()
options.add_argument("start-maximized")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
driver = webdriver.Chrome(options=options, executable_path=r'C:\WebDrivers\chromedriver.exe')
driver.get('https://www.imdb.com/list/ls000004717/')
driver.execute_script("return arguments[0].scrollIntoView(true);", WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.CSS_SELECTOR, "span.pagination-range"))))
while True:
try:
WebDriverWait(driver, 20).until(EC.invisibility_of_element((By.CSS_SELECTOR, "div.row.text-center.lister-working.hidden")))
driver.execute_script("return arguments[0].scrollIntoView(true);", WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.CSS_SELECTOR, "span.pagination-range"))))
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "a.flat-button.lister-page-next.next-page"))).click()
print("Clicked on NEXT button")
except TimeoutException as e:
print("No more NEXT button")
break
driver.quit()
Console Output:
Clicked on NEXT button
Clicked on NEXT button
Clicked on NEXT button
Clicked on NEXT button
Clicked on NEXT button
Clicked on NEXT button
Clicked on NEXT button
Clicked on NEXT button
Clicked on NEXT button
No more NEXT button