I am trying to scrape the Google News page in the following way:
from selenium import webdriver
import time
from pprint import pprint
base_url = 'https://www.google.com/'
driver = webdriver.Chrome('/home/vincent/wintergreen/chromedriver') ## change here to your location of the chromedriver
driver.implicitly_wait(30)
driver.get(base_url)
input = driver.find_element_by_id('lst-ib')
input.send_keys("brexit key dates timetable schedule briefing")
click = driver.find_element_by_name('btnK')
click.click()
news = driver.find_element_by_link_text('News')
news.click()
tools = driver.find_element_by_link_text('Tools')
tools.click()
time.sleep(1)
recent = driver.find_element_by_css_selector('div.hdtb-mn-hd[aria-label=Recent]')
recent.click()
# custom = driver.find_element_by_link_text('Custom range...')
custom = driver.find_element_by_css_selector('li#cdr_opt span')
custom.click()
from_ = driver.find_element_by_css_selector('input#cdr_min')
from_.send_keys("9/1/2018")
to_ = driver.find_element_by_css_selector('input#cdr_max')
to_.send_keys("9/2/2018")
time.sleep(1)
go_ = driver.find_element_by_css_selector('form input[type="submit"]')
print(go_)
pprint(dir(go_))
pprint(go_.__dict__)
go_.click()
This script manage to enter search terms, switch to the news tab, open the custom time period tab, fill in start and end date, but fails to click on the 'Go' button after that point.
From the print and pprint statement at the end of the script, I can deduct that it does find the 'go' button succesfully, but is somehow unable to click on it. The error displays as selenium.common.exceptions.ElementNotVisibleException: Message: element not visible
Could anyone experienced with Selenium have a quick run at it and give me hints as why it returns such error?
Thx!
Evaluating the css using developer tools in chrome yields 4 elements.
Click here for the image
use the following css instead:
go_ = driver.find_element_by_css_selector('#cdr_frm > input.ksb.mini.cdr_go')
Related
I am trying to select dropdown year of 2021 on (https://www.theknot.com/registry/couplesearch) and am unable to figure out how to use the dropdown.
#This code is working
typetextfirst = driver.find_element_by_id("couples-search-first-name")
typetextfirst.clear()
typetextfirst.send_keys(row["First"])
typetextlast = driver.find_element_by_id("couples-search-last-name")
typetextlast.clear()
typetextlast.send_keys(row["Last"])
typetextyear = driver.find_element_by_id("couples-search-year")
#None of these options work to populate the year
typetextyear.selectByIndex(1)
typetextyear.select_by_index(1)
typetextyear.selectByVisibleText("2021")
typetextyear.select_by_visible_text("2021")
#This code is working
typetextlast.send_keys(Keys.ENTER)
Page doesn't use standard dropdown widget but it uses button and ul to emulate dropdown.
This code works for me on Firefox and Chrome on Linux Mint.
First I click button to open dropdown created with ul and later I search li with expected text and click it.
Because it may have text 2021 with some spaces/tabs/enters (which browser doesn't show) so I prefer contains instead of =
from selenium import webdriver
url = 'https://www.theknot.com/registry/couplesearch'
driver = webdriver.Firefox()
#driver = webdriver.Chrome()
driver.get(url)
year_dropdown = driver.find_element_by_id("couples-search-year")
year_dropdown.click()
year = year_dropdown.find_element_by_xpath(".//li[contains(text(), '2021')]")
#year = year_dropdown.find_element_by_xpath(".//li[text()='2021']")
year.click()
I'am new to mobile automation. I thought I got the gist, but I ran into a problem.
An error occurs when using the command send_keys()
selenium.common.exceptions.InvalidElementStateException: Message: Cannot set the element to '321785214'. Did you interact with the correct element?
I have a code
driver = webdriver.Remote("http://localhost:4723/wd/hub", desired_cap)
# Skip fingerprint
driver.find_element_by_xpath("//android.widget.Button[#bounds='[413,1802][668,1877]']").click()
# Click Continue
driver.find_element_by_xpath("//android.widget.Button[#content-desc='Continue']").click()
# Click to input phone number
input_phone_number = driver.find_element_by_xpath("//android.widget.EditText").click()
input_phone_number_edit = driver.find_element_by_class_name("android.widget.EditText")
# Enter phone number
input_phone_number_edit.send_keys("321785214")
And I have code from other mobile app and this code is working
driver = webdriver.Remote("http://localhost:4723/wd/hub", desired_cap)
# click to Set up later button
set_up_later_btn = driver.find_element_by_xpath('//android.widget.Button[#content-desc="Set up later"]').click()
input_phone_number = driver.find_element_by_xpath('//android.view.View[#content-desc="Phone number"]').click()
input_phone_number_edit = driver.find_element_by_class_name('android.widget.EditText')
driver.implicitly_wait(50)
input_phone_number_edit.send_keys('178546128')
driver.find_element_by_accessibility_id("Continue").click()
I don't know why first code isn't working. I need help. Thanks
Instead of using the default send_keys method, you can use the send_keys method provided by the ActionChains class
from selenium.webdriver import ActionChains
actions = ActionChains(self.driver)
actions.send_keys("321785214")
actions.perform()
I am trying to webscrape indeed.com to search for jobs using python, with selenium and beautifulsoup. I want to click next page but cant seem to figure out how to do this. Looked at many threads but it is unclear to me which element I am supposed to perform on. Here is the web page html and the code marked with grey comes up when I inspect the next button.
Also just to mention I tried first to follow what happens to the url when mousedown is executed. After reading the addppurlparam function and adding the strings in the function and using that url I just get thrown back to page one.
Here is my code for the class with selenium meant to click on the button:
from selenium import webdriver
from selenium.webdriver import ActionChains
driver = webdriver.Chrome("C:/Users/alleballe/Downloads/chromedriver.exe")
driver.get("https://se.indeed.com/Internship-jobb")
print(driver.title)
#assert "Python" in driver.title
elem = driver.find_element_by_class_name("pagination-list")
elem = elem.find_element_by_xpath("//li/a[#aria-label='Nästa']")
print(elem)
assert "No results found." not in driver.page_source
assert elem
action = ActionChains(driver).click(elem)
action.perform()
print(elem)
driver.close()
The indeed site is formatted so that it shows 10 per page.
Your photo shows the wrong section of HTML instead you can see the links contain start=0 for the first page, start=10 for the second, start=20 for the third,...
You could use this knowledge to do a code like this:
while True:
i = 0
driver.get(f'https://se.indeed.com/jobs?q=Internship&start={i}')
# code here
i = i + 10
But, to directly answer to your question you should do:
next_page_link = driver.find_element_by_xpath('/html/head/link[6]')
driver.get(next_page_link)
This will find the link and then get it.
its work. paginated to next page.
driver.find_element_by_class_name("pagination-list").find_element_by_tag_name('a').click()
I have been trying for the past couple of days to select a dropdown and at least print out the options available, but I just cannot get it to work.
I am getting the this error when I run the module.
Traceback (most recent call last):
File "sel_test_elements2.py", line 20, in
print ([o.text for o in select_element.options])
AttributeError: 'FirefoxWebElement' object has no attribute 'options'
Currently my code looks like this.
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.by import By
# Define Global Variables
url = "https://games.pcaha.ca/teams/4329"
csv_file = "game_schedule_4329.csv"
games = []
# create a new Firefox session
driver = webdriver.Firefox()
driver.get(url)
driver.implicitly_wait(30)
# Locate the Sector and create a Select object
select_element = driver.find_element_by_css_selector(".team-filters")
# this will print out strings available for selection on select_element, used in visible text below
print ([o.text for o in select_element.options])```
The issue you face is the fact that this website is using react and doesn't use default Select and Options. They have a custom dropdown implemented, so the way to interact with it is the same as interaction with regular web elements, Select and Options won't work in this case.
I modified your code and it works for me in Chrome:
from selenium.webdriver import Chrome
from time import sleep
# Define Global Variables
url = "https://games.pcaha.ca/teams/4329"
csv_file = "game_schedule_4329.csv"
games = []
# create a new Chrome session
driver = Chrome()
driver.get(url)
driver.implicitly_wait(30)
sleep(3) # make sure svgs load before interaction
# Click on arrow down
arrow = driver.find_elements_by_css_selector(".team-filters svg")[1].click()
# Collect options
options = driver.find_elements_by_xpath("//div[contains(#id, 'react-select-2')]")
# Print text from options
print([o.text for o in options])
Note: when manually opening the dropdown in your browser and trying to use web inspector, it closes, so in order to get the html inside a dropdown, you can use something like:
dropdown = driver.find_element_by_css_selector("div.css-kj6f9i-menu")
dropdown_html = dropdown.get_attribute('innerHTML')
I hope it helped. Good luck!
I have used something similar in a small script i have written, may be it can give you an hint on how to go about
Approach 1 This is to select the last of the options available
Variable options in the code below gets be the option available for the dropdown
select_datebox = driver.find_element_by_id('jrnyDateSrchTxt') # Drop down selection, you have to change the id appropriately
select_datebox.click()
time.sleep(2)
options = select_datebox.find_elements_by_tag_name('option')
options[len(options)-1].click() #selecting the last option
Approach 1 entering the option via a variable
select = Select(driver.find_element_by_id("jrnyDateSrchTxt")) # Drop down selection, you have to change the id appropriately
time.sleep(1)
select.select_by_value(datadate) # Date selection
time.sleep(2)
I am trying to automate a booking in process on a travel site using
splinter and having trouble clicking on a css element on the page.
This is my code
import splinter
import time
secret_deals_email = {
'user[email]': 'adf#sad.com'
}
browser = splinter.Browser()
url = 'http://roomer-qa-1.herokuapp.com'
browser.visit(url)
click_FIND_ROOMS = browser.find_by_css('.blue-btn').first.click()
time.sleep(10)
# click_Book_button = browser.find_by_css('.book-button-row.blue-btn').first.click()
browser.fill_form(secret_deals_email)
click_get_secret_deals = browser.find_by_name('button').first.click()
time.sleep(10)
click_book_first_room_list = browser.find_by_css('.book-button-row-link').first.click()
time.sleep(5)
click_book_button_entry = browser.find_by_css('.entry-white-box.entry_box_no_refund').first.click()
The problem is whenever I run it and the code gets to the page where I need to click the sort of purchase I would like. I can't click any of the option on the page.
I keep getting an error of the element not existing no matter what should I do.
http://roomer-qa-1.herokuapp.com/hotels/atlanta-hotels/ramada-plaza-atlanta-downtown-capitol-park.h30129/44389932?rate_plan_id=1&rate_plan_token=6b5aad6e9b357a3d9ff4b31acb73c620&
This is the link to the page that is causing me trouble please help :).
You need to whait until the element is present at the website. You can use the is_element_not_present_by_css method with a while loop to do that
while not(is_element_not_present_by_css('.entry-white-box.entry_box_no_refund')):
time.sleep(50)