I am trying to scrape a website and fairly new to Python. I have managed to come up with below code. The problem however is it goes into an infinite loop after reaching the last page i.e Next button is greyed out.
Also i don't think i am catching the Stale Element properly here. Any help would be greatly appreciated!`
pages_remaining = True
while pages_remaining:
button=driver.find_element_by_class_name("arrow-right")
href_data = button.get_attribute('href')
if href_data is not None:
soup=BeautifulSoup(driver.page_source,"html.parser")
data = soup.find_all("div",{"class":"shelfProductStamp-content row"})
count = 1
for item in data:
ProductText=item.find("a",attrs={"class":"shelfProductStamp-imageLink"})["title"]
if item.find("span",attrs={"class":"sf-pricedisplay"}) is not None:
Price=item.find("span",attrs={"class":"sf-pricedisplay"}).text
else:
Price=""
if item.find("p",attrs={"class":"sf-comparativeText"}) is not None:
SubPrice1=item.find("p",attrs={"class":"sf-comparativeText"}).text
else:
SubPrice1=""
if item.find("span",attrs={"class":"sf-regoption"}) is not None:
Option=item.find("span",attrs={"class":"sf-regoption"}).text
else:
Option=""
SubPrice=str(SubPrice1)+"-"+str(Option)
SaleDates=item.find("div",attrs={"class":"sale-dates"}).text
urll2=driver.current_url
PageNo=driver.find_element_by_class_name("current").text
writer.writerow([ProductText,Price,SubPrice,SaleDates,PageNo])
count+=1
try:
def find(driver):
element = driver.find_element_by_class_name("arrow-right")
if element:
return element
else:
pages_remaining=False
#driver.quit()
time.sleep(10)
driver.implicitly_wait(10)
element = WebDriverWait(driver, 60).until(find)
driver.execute_script("arguments[0].click();", element)
except StaleElementReferenceException:
pass
else:
break
Thanks
When you set pages_remaining = False inside the find() function, that is a local variable. It is not the same variable as pages_remaining in the outer loop.
If you want to do it that way, you'll need to make it a global.
Thanks for your help here. I managed to fix this by simply adding another if statement at the end and swapping the time.sleep(10) as below
try:
def find(driver):
element = driver.find_element_by_class_name("arrow-right")
if element:
return element
driver.implicitly_wait(10)
element = WebDriverWait(driver, 60).until(find)
driver.execute_script("arguments[0].click();", element)
time.sleep(10)
except StaleElementReferenceException:
pass
if href_data is None:
break
Related
i wanna know if element appear then disappear i want to apply it in this code
i know that internet is back by a text appear when internet is gone or its back
except:
time.sleep(3)
if driver.find_element(By.CLASS_NAME, "KhLQZTRq").size['width'] != 0 :
print ('internet lost')
while True:
time.sleep(5)
if driver.find_element(By.CLASS_NAME, "KhLQZTRq").size['width'] == 0 :
print("internet back")
else:
continue
else:
print("nothing happen")
This is working fine:
if driver.find_element(By.CLASS_NAME, "KhLQZTRq").size['width'] != 0
and it print 'internet lost' when the the element appear
But this is not working:
if driver.find_element(By.CLASS_NAME, "KhLQZTRq").size['width'] == 0
i just want it to print " internet back " if the element is no longer present
If the element appears or disappears you could write
if driver.find_element(By.CLASS_NAME, "KhLQZTRq"):
It will return True if the element exists and False if it doesn't
Trying to pick out single restaurant elements under the All Restaurants category on this page. https://www.foodpanda.sg/restaurants/new?lat=1.2915902&lng=103.8379066&vertical=restaurants
All li elements have a different class name. The only working xpath, I have been able to figure out is this one.
//ul[#class="vendor-list"]//li[3]
I cannot figure out how to increase this number and get all the restaurants, which is complicated by the fact that there is an infinite scroll as well.
j = 1
def get_rest():
global j
while True:
driver.execute_script("window.scrollBy(0,2525)", "")
time.sleep(5)
var = f'{[j]}'
elems = driver.find_elements_by_xpath(f'//ul[#class="vendor-list"]//li{var})
return elems
I guess it should be something like this
all_restaurants = set()
restaurant_locator = '//ul[#class="vendor-list"]//li[#data-testid and not(#class)]'
page_bottom_locator = '.restaurants__city-bottom-info'
while #bottom is not reached:
restaurants = driver.find_elements_by_xpath(restaurant_locator)
all_restaurants.add(restaurants)
time.sleep(1)
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
I tried to get the innerHTML of the //ul[#class="vendor-list"]//li
Code :
driver.get("https://www.foodpanda.sg/restaurants/new?lat=1.2915902&lng=103.8379066&vertical=restaurants")
while True:
for item in driver.find_elements(By.XPATH, "//ul[#class='vendor-list']/li"):
ActionChains(driver).move_to_element(item).perform()
sleep(0.1)
print(item.get_attribute('innerHTML'))
I am trying to trigger an action when the number 1 appears on a web element, but how do I check for it?
This is what I am trying to do, and I get the error 'WebElement' is not iterable
def is_0(self):
nav = Chrome()
nav.set_window_size(1360, 760)
while True:
if 1 in nav.find_element_by_xpath('//*[#id="header"]/nav/ul/li[4]/a/span/b'):
break
else:
print('Verificando')
sleep(2)
In order to get a text of an element, you should use ".text" or ".get_attribute('innerHTML')". I used ".get_attribute('innerHTML')" in my code.
Please refer to my code.
def is_0(self):
nav = Chrome()
nav.set_window_size(1360, 760)
while True:
if '1' in nav.find_element_by_xpath('//*[#id="header"]/nav/ul/li[4]/a/span/b').get_attribute('innerHTML'):
break
else:
print('Verificando')
sleep(2)
I want to know how it can refresh until the element is found, for some reason this code wont work, it just right through the loop.
while True:
try:
element = driver.find_elements_by_xpath('//button[text()="SIGN UP"]')
except NoSuchElementException:
driver.refresh()
else:
break
driver.find_element_by_xpath('//button[text()="SIGN UP"]')
print("FOUND")
I haven't used selenium before but I'm going to make a safe bet that if the driver doesn't find anything matching that xpath, it will simply return None. In other words, your code is still valid because element is simply a None, thus not raising the error. What you can do instead is use a while loop to keep refreshing until it finds the element. Adding this function to a separate thread or process wouldn't be a bad idea either in case you have other things happening in the main loop.
def retry_until_id_found(element_id, max_retries=100):
retries = 0
while not driver.find_element_by_xpath(element_id)
if retries <= max_retries:
time.sleep(3)
driver.refresh()
retries+=1
else:
print(f"ELEMENT {element_id} NOT FOUND")
return None
print(f"FOUND {element_id}")
return driver.find_element_by_xpath(element_id)
element = retry_until_id_found(r'//button[text()="SIGN UP"]')
I believe below logic will help you
`if(element.visible) //false
{
driver.refresh()
n = 5 // you can give end limit
for(i=0;i<n;i++)
{
if(element.visible) //true
{
driver.find_element_by_xpath('//button[text()="SIGN UP"]')
print("FOUND")
i=n; //break the loop
}
else
{
driver.refresh()
}
}
}`
I am going through the table at the following link:
http://cancer.sanger.ac.uk/cosmic/sample/overview?id=2120881
through selenium in python. This is the code:
driver = webdriver.Chrome()
driver.get('http://cancer.sanger.ac.uk/cosmic/sample/overview?id=2120881')
elem = driver.find_element_by_link_text("Variants")
while elem:
elem.click()
time.sleep(5)
try:
elem = driver.find_element_by_link_text("Next")
print(elem.is_enabled())
if 'disabled' in elem.get_attribute('class'):
break
except:
print('Next not available or page not loaded!')
driver.quit()
I have trouble changing the number of displayed values to 100. How would I do that?
Also, why does the is_enabled() return True even when the button becomes unclickable?
Thanks in advance!
Q: I have trouble changing the number of displayed values to 100. How would I do that?
Solution: You have to use Select Class (link) to select value from Dropdown
Question:why does the is_enabled() return True even when the button becomes unclickable?
The is_enabled() is returning True only when it is as you can see the True is only printed three times when it was enabled and post that its enter in break loop and exits as expected
Just updated your code with Select and break print statement:
from selenium import webdriver
import time
from selenium.webdriver.support.ui import Select
driver = webdriver.Chrome()
driver.maximize_window()
driver.get('http://cancer.sanger.ac.uk/cosmic/sample/overview?id=2120881')
elem = driver.find_element_by_link_text("Variants")
while elem:
elem.click()
time.sleep(5)
select = Select(driver.find_element_by_name('DataTables_Table_0_length'))
select.select_by_value('100') # you can use any value
try:
elem = driver.find_element_by_link_text("Next")
print(elem.is_enabled())
if 'disabled' in elem.get_attribute('class'):
print "Before Break"
break
except:
print('Next not available or page not loaded!')
driver.quit()
The output is:
True
True
True
Before Break