I am trying to get some data from Google but it I get the "before you continue" google popup. I am trying to make selenium locate the button and click it and return to the getting data but it seems even if I have the button ID in the code it doesn't find it
"""
Search on Google and returns the list of PAA questions in SERP.
"""
def newSearch(browser,query):
if lang== "en":
browser.get("https://www.google.com?hl=en")
WebDriverWait(browser, 10).until(EC.frame_to_be_available_and_switch_to_it((By.XPATH, "//iframe")))
agree = WebDriverWait(browser, 10).until(EC.element_to_be_clickable((By.XPATH, '//*[#id="L2AGLb"]/span/span')))
agree.click()
browser.switch_to_default_content()
searchbox = browser.find_element_by_xpath("//input[#aria-label='Search']")
else:
browser.get("https://www.google.com?hl=es")
searchbox = browser.find_element_by_xpath("//input[#aria-label='Buscar']")
searchbox.send_keys(query)
sleepBar(2)
tabNTimes()
if lang== "en":
searchbtn = browser.find_elements_by_xpath("//input[#aria-label='Google Search']")
else:
searchbtn = browser.find_elements_by_xpath("//input[#aria-label='Buscar con Google']")
try:
searchbtn[-1].click()
except:
searchbtn[0].click()
sleepBar(2)
paa = browser.find_elements_by_xpath("//span/following-sibling::div[contains(#class,'match-mod-horizontal-padding')]")
hideGBar()
return paa
Try clicking the inner div of the button itself. HTML of the agree popup:
<button id="L2AGLb" class="tHlp8d" data-ved="0ahUKEwj89p7Swob1AhVBxhoKHS0gDxIQiZAHCCE">
<div class="QS5gu sy4vM" role="none">
Acepto
</div>
</button>
Your selector should look like this:
(By.CSS_SELECTOR, "#L2AGLb > div")
Here is a working full example:
def test_google_...(self):
driver = self.driver
if self.LANGUAGE == "en":
driver.get("https://www.google.com?hl=en")
else:
driver.get("https://www.google.com?hl=es")
WebDriverWait(driver, 5).until(
EC.presence_of_element_located((By.CSS_SELECTOR, "#L2AGLb > div"))
)
driver.find_element(By.CSS_SELECTOR, "#L2AGLb > div").click()
WebDriverWait(driver, 5).until(
EC.presence_of_element_located((By.CSS_SELECTOR, 'input[name="q"]'))
)
query = "your search query"
driver.find_element(By.CSS_SELECTOR, 'input[name="q"]').send_keys(query)
driver.find_element(By.CSS_SELECTOR, 'input[name="q"]').send_keys(Keys.RETURN)
...
Related
I am scraping numerous similar webpages, however; to get the same information lies beneath different XPATHs on some of the pages.
Here are the two XPATHs that I am tryin to alternate.
city_e = WebDriverWait(driver, 20).until(
EC.presence_of_element_located((By.XPATH, "//div/h4"))
)
alternative_name_e = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.XPATH, "//*[#id='SFbizctc53fcd34ec260b1442c7bd7b4']/div/div[1]"))
)
FULL CODE BELOW:
results = []
for i in range(6):
links = WebDriverWait(driver, 60).until(
EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".SFpne a"))
)
links[i].click()
time.sleep(4)
name_e = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.XPATH, "//div/h3"))
)
alternative_name_e = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.XPATH, "//*[#id='SFbizctc53fcd34ec260b1442c7bd7b4']/div/div[1]"))
)
city_e = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.XPATH, "//div/h4"))
)
jobTitle_e = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.XPATH, "//div[#itemprop='contactPoint']/div"))
)
address_e = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.XPATH, "//div[#itemprop='contactPoint']/address"))
)
cell_e = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.XPATH, "//div/a[#class='SFbizctcphn']"))
)
email_e = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.XPATH, "//*[#id='SFbizpne0']/div[3]/div/a"))
)
full_member_content = {
'Member_Name': [alternative_name_e.text],
'member_Name': [name_e.text],
'member_City': [city_e.text],
'member_JobTitle': [jobTitle_e.text],
'member_Address': [address_e.text],
'member_Cell': [cell_e.text],
'member_Email': [email_e.text]
}
results.append(full_member_content)
time.sleep(4)
driver.back()
print(results)
Just curios if there is a try catch of an if statement that I could add to accomplish this.
Second Challenge: If this is an easy fix, I am also curious of whether I could re-run city_e= WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div/h4"))) when more that one h4 tag appears.
Thank you!
Error Message
You can't look for non existance of an object. For your first question you're looking at:
Forcefully sleeping.
time.sleep(4)
Then doing something similar to what you did with links
exists =EC.presence_of_all_elements_located((your selector))
Before doing anything check the length of exists. If it's more than 0 it is there otherwise it is the other one.
For your bonus question do the same thing with all elements located for the h4 tag and go over them one by one just like youre doing now with links
hello im practicing selenium on a practice forum this the link for it :
click here
if visit the page and inspect element on the dropdown menu for state and city you will find it consists of only div element i tried doing this but didnt work obviously :
dropdown = Select(d.find_element("xpath",'//*[#id="state"]'))
dropdown.select_by_index(0)
this the error message :
Select only works on <select> elements, not on <div>
can someone show how to loop through the value of the menu or is there any other solution ?
This code is working
search_url = 'https://demoqa.com/automation-practice-form'
driver = webdriver.Chrome(options = options, executable_path= os.path.join(os.environ['USERPROFILE'],"Desktop") + f'\\Python\\available Tender\\chromedriver\\chromedriver.exe')
driver.get(search_url)
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
element1 = WebDriverWait(driver, 4).until(EC.presence_of_element_located((By.XPATH, f"//div[#id='adplus-anchor']")))
driver.execute_script("""
var element = arguments[0];
element.parentNode.removeChild(element);
""", element1)
element2 = WebDriverWait(driver, 4).until(EC.presence_of_element_located((By.XPATH, f"//div[#id='currentAddress-wrapper']")))
driver.execute_script("""
var element = arguments[0];
element.parentNode.removeChild(element);
""", element2)
driver.find_element(By.XPATH, '//*[#id="state"]/div/div[2]/div').click()
e1 = WebDriverWait(driver, 4).until(EC.presence_of_element_located((By.XPATH, f"//div[contains(#class,'menu')]")))
e1.find_element(By.XPATH, "//div[contains(text(),'NCR')]").click()
I am trying to get the name of the restaurant and the address from this website: [https://www.kravekar.com/restaurants][webPage]
The issue is that each time I return to the main page, I get this error:
Element <div class="restaurant-card">...</div> is not clickable at point (1129, 435). Other element would receive the click: <i class="fa fa-spinner fa-pulse"></i>
I tried to implement a driver refresh, a time sleep but is not working. I got the same error in the third iteration.
So far this is my reproducible code:
driver.get('https://www.kravekar.com/restaurants')
comment_button = driver.find_elements(by =By.CSS_SELECTOR, value = "div.restaurant-card")
result = []
for btn in comment_button :
btn.click()
try:
name = driver.find_element(by=By.XPATH, value = '//*.
[#id="restaurant_menu_head"]/div/div/div[2]/div[1]/div/div/h4')
name = name.text
print(name)
address = driver.find_element(by = By.XPATH, value = '//*
[#id="restaurant_menu_head"]/div/div/div[2]/div[1]/div/div/div/span')
address = address.text
print(address)
except:
print("No address or name")
driver.execute_script("window.history.go(-1)")
When you do btn.click() or driver.execute_script("window.history.go(-1)") it might be possible that the reference to the correct webpage is lost. So it is better to store the url of all the restaurants right from the home page, and then loop over the stored urls.
driver.get('https://www.kravekar.com/restaurants')
cards = driver.find_elements(By.CSS_SELECTOR, ".restaurant-card-wrapper")
urls = [card.get_attribute('href') for card in cards]
names = [card.find_element(By.CSS_SELECTOR, ".restaurant-card-title").text for card in cards]
for idx,url in enumerate(urls):
try:
driver.get(url)
# name = driver.find_element(By.CSS_SELECTOR, "#tab_menu_info h4.media-heading").text
print(names[idx])
address = driver.find_element(By.CSS_SELECTOR, "#tab_menu_info .restaurant_menu_info-addresss").text
print(address)
except:
print("No address or name")
which outputs
Arby's
51171 Highway 6 & 24, Glenwood Springs, CO 81601
Springs Bar and Grill
722 Grand Ave, Glenwood Springs, CO 81601
etc.
After starting the scraper something strange happens: it either works properly, ends after visiting the second page and clicking the Next button, or it somehow ends up on the property page, when I use the code with the line that is currently commented out. However, when that line is placed as it is now, it seems to work, it visits all the pages and scrapes them, and eventually, I get a time out. I am unsure what the issue is? Any tips?
The current code:
class PropertyFoxSpider(scrapy.Spider):
name = 'property_fox'
start_urls = [
'https://propertyfox.co.za/listing-search?currentpage=1&term_id=62515&keywords=Western+Cape&orderby=createddate:desc&status%5B%5D=Active'
]
def __init__(self):
#path to driver
self.driver = webdriver.Chrome('my_path')
def parse(self,response):
url = self.driver.get(response.url)
while True:
WebDriverWait(self.driver, 10).until(lambda driver: self.driver.current_url != url)
try:
elem = WebDriverWait(self.driver, 10).until(EC.element_to_be_clickable((By.ID, "pagerNext")))
elem.click()
#WebDriverWait(self.driver, 10).until(lambda driver: self.driver.current_url != url)
url = self.driver.current_url
yield scrapy.Request(url=url, callback=self.parse_page, dont_filter=False)
except TimeoutException:
break
def parse_page(self, response):
for prop in response.css('div.property-item'):
link = prop.css('a::attr(href)').get()
banner = prop.css('div.property-figure-icon div::text').get()
sold_tag = None
if banner:
banner = banner.strip()
sold_tag = 'sold' if 'sold' in banner.lower() else None
yield scrapy.Request(
link,
callback=self.parse_property,
meta={'item': {
'agency': self.name,
'url': link,
'offering': 'buy',
'banners': banner,
'sold_tag': sold_tag,
}},
)
def parse_property(self, response):
item = response.meta.get('item')
...
It seem that Selenium "recognize" even disabled Next button as clickable element and still tries to click it even on last page. You can try below code to make it work:
def parse(self,response):
self.driver.get(response.url)
url = self.driver.current_url
while True:
try:
elem = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//span[#id="pagerNext" and not(#class="disabled")]')))
elem.click()
except TimeoutException:
break
WebDriverWait(self.driver, 10).until(lambda driver: self.driver.current_url != url)
url = self.driver.current_url
yield scrapy.Request(url=url, callback=self.parse_page, dont_filter=False)
Note that I replaced (By.ID, "pagerNext") locator with (By.XPATH, '//span[#id="pagerNext" and not(#class="disabled")]'), so now only enabled Next button will be clicked
After clicking "Advanced Search" on page https://www.tmdn.org/tmview/welcome#
I want to select :
Designated territories
Trade mark offices
Trade mark status
Application date
Below is the sample script which will click on Advanced Search button and select Japan from Designated Territories option.
SampleScript.py:
import Locators
from selenium import webdriver
url = 'https://www.tmdn.org/tmview/welcome#'
driver = None
# Initialize Chrome driver
driver = webdriver.Chrome()
driver.get(url)
driver.implicitly_wait(10)
AdvancedSearchElement = driver.find_element(*Locators.AdvancedSearchElement)
AdvancedSearchElement.click()
TerritoryDropDownElement = driver.find_element(*Locators.TerritoryDropDownElement )
TerritoryDropDownElement .click()
TerritoryLabelElements = driver.find_elements(*Locators.TerritoryLabelsElement)
for elem in TerritoryLabelElements:
print(elem.text)
if elem.text == 'Japan':
elem.click()
print('Selected Japan')
TerritoryDropDownElement .click()
Locators.py:
AdvancedSearchElement = (By.CSS_SELECTOR, '#lnkAdvancedSearch')
TerritoryDropDownElement = (By.ID, 'DesignatedTerritories')
TerritoryLabelsElement = (By.CSS_SELECTOR, 'div.multiSelectOptions label')