I'm trying to get some info from pages that look like this link to the site
I need to scroll all the way down, I wrote a function for that.
def Scroll():
startPos = driver.execute_script("return window.pageYOffset;")
while 1:
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);") #need to wait
time.sleep(0.1)
newPos = driver.execute_script("return window.pageYOffset;")
if newPos == startPos:
break
startPos = newPos
return
I would like to change time.sleep(0.1) with some conditional wait until the line before that one is executed, is there a way to do this? There is no special element loaded after every scroll or at the end of the page.
Solution:
...
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
while 1:
if driver.execute_script("return jQuery.active == 0"):
break
else:
time.sleep(0.05)
continue
...
Instead of thread.sleep, you can use following code which waits for Ajax calls to complete because every scroll, there will be a Ajax call in the background.
WebDriverWait wait=new WebDriverWait(driver, 120);
Wait.until(ExpectedConditions.jsReturnValue("return jQuery.active == 0"));
Related
from selenium import webdriver
from time import sleep
from selenium.webdriver.common.by import By
chrome_options = webdriver.ChromeOptions()
driver = webdriver.Chrome(r'chromedriver.exe', options=chrome_options)
url = 'https://rarible.com/connect'
b = 1
def main():
driver.get(url)
input('Login..(Press ENTER when finsihed)')
sleep(1)
while driver.current_url == "https://rarible.com/":
ts_href = '/html/body/div[1]/div[2]/div[2]/div[2]/div/div/div/div[4]/div[2]/div/div[1]/div[1]/div/div/div[' \
'1]/div[1]/div/div[3]/a'
href = driver.find_element(By.XPATH, ts_href).get_attribute('href')
driver.get(href)
sleep(2)
followers = '/html/body/div/div[2]/div[2]/div[2]/div[2]/div/div/div/div[1]/div[5]/div/button[1]/span[2]'
driver.find_element(By.XPATH, followers).click()
sleep(3)
# buttons = driver.find_elements(By.XPATH, '//button[normalize-space()="Follow"]')
def butts():
global b
fbtn = f'/html/body/div/div[1]/div/div[2]/div/div[2]/div/div/div/div[{b}]/div/div/div[3]/button'
buttons = driver.find_element(By.XPATH, fbtn)
print(f'BUTTON {b} TEXT (FOLLOW/UNFOLLOW): {buttons.text}')
if buttons.text == "Follow":
buttons.click()
b += 1
sleep(1)
butts()
elif buttons.text == "Unfollow":
b += 1
butts()
butts()
print('All set here, onto the next one...')
else:
driver.get('https://rarible.com/')
if __name__ == '__main__':
main()
I cannot get it to click the follow buttons.
I Can't find an iframe that they are hiding in, or any other type of javascript voodoo being done but i am not the most experienced. Which is why i come to you seeking guidance.
The line where you define followers wasn't working for me so I changed it to
followers = '//button[#datamarker="root/appPage/address/profile/followingLinks/followers"]'
Trying to figure out how to get the butts() loop to run, but I don't have any followers on rarible so it's kind of hard to test. However, putting these lines of code in at the end of "if buttons.text == "Follow": before butts() is called again might work.
scroll_div = driver.find_element(By.XPATH, '/html/body/div/div[1]/div/div[2]/div/div[2]/div/div/div')
scroll_div.send_keys(Keys.ARROW_DOWN)
It's not actually an iframe. The solution has something to do with scrolling through the follow buttons
Trying to pick out single restaurant elements under the All Restaurants category on this page. https://www.foodpanda.sg/restaurants/new?lat=1.2915902&lng=103.8379066&vertical=restaurants
All li elements have a different class name. The only working xpath, I have been able to figure out is this one.
//ul[#class="vendor-list"]//li[3]
I cannot figure out how to increase this number and get all the restaurants, which is complicated by the fact that there is an infinite scroll as well.
j = 1
def get_rest():
global j
while True:
driver.execute_script("window.scrollBy(0,2525)", "")
time.sleep(5)
var = f'{[j]}'
elems = driver.find_elements_by_xpath(f'//ul[#class="vendor-list"]//li{var})
return elems
I guess it should be something like this
all_restaurants = set()
restaurant_locator = '//ul[#class="vendor-list"]//li[#data-testid and not(#class)]'
page_bottom_locator = '.restaurants__city-bottom-info'
while #bottom is not reached:
restaurants = driver.find_elements_by_xpath(restaurant_locator)
all_restaurants.add(restaurants)
time.sleep(1)
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
I tried to get the innerHTML of the //ul[#class="vendor-list"]//li
Code :
driver.get("https://www.foodpanda.sg/restaurants/new?lat=1.2915902&lng=103.8379066&vertical=restaurants")
while True:
for item in driver.find_elements(By.XPATH, "//ul[#class='vendor-list']/li"):
ActionChains(driver).move_to_element(item).perform()
sleep(0.1)
print(item.get_attribute('innerHTML'))
I am trying to trigger an action when the number 1 appears on a web element, but how do I check for it?
This is what I am trying to do, and I get the error 'WebElement' is not iterable
def is_0(self):
nav = Chrome()
nav.set_window_size(1360, 760)
while True:
if 1 in nav.find_element_by_xpath('//*[#id="header"]/nav/ul/li[4]/a/span/b'):
break
else:
print('Verificando')
sleep(2)
In order to get a text of an element, you should use ".text" or ".get_attribute('innerHTML')". I used ".get_attribute('innerHTML')" in my code.
Please refer to my code.
def is_0(self):
nav = Chrome()
nav.set_window_size(1360, 760)
while True:
if '1' in nav.find_element_by_xpath('//*[#id="header"]/nav/ul/li[4]/a/span/b').get_attribute('innerHTML'):
break
else:
print('Verificando')
sleep(2)
I am implementing a while loop in selenium, and want to condition my while loop, so when the scroll bar is at its end of its scroll the while loop should stop. How can i code this type of condition in while loop?
Iam using Keys.DOWN and my while loop is right now set to True
My code of while loop:
while True:
self.driver.find_element_by_id('pane-side').send_keys(Keys.DOWN * 5)
self.driver.find_elements_by_xpath("//div[#class='_2wP_Y']")
Here is the pseudo code that should do the trick.
Check if the current (scrolled position+ window height) is greater than page height(-1)
pageHeight = driver.execute_script("return document.body.scrollHeight")
totalScrolledHeight = driver.execute_script("return window.pageYOffset + window.innerHeight")
# -1 is to make sure the rounding issues
if((pageHeight-1)<=totalScrolledHeight):
print("pass")
else:
print("Failed")
I think this will work for you.
It checks if the new height is greater than the previous height. If it's True, it will continue. Else, it will break the loop.
pre_height = 0
new_height = 0
while True:
time.sleep(1)
self.driver.execute_script("window.scrollBy(0, 5000);")
new_height = self.driver.execute_script("return document.body.scrollHeight")
if pre_height < new_height:
print('once again')
pre_height = self.driver.execute_script("return document.body.scrollHeight")
else:
print("maybe finished?")
break
Set the condition of the while loop to check if the position of the scroll bar is at the lowest point possible.
I am trying to scrape a website and fairly new to Python. I have managed to come up with below code. The problem however is it goes into an infinite loop after reaching the last page i.e Next button is greyed out.
Also i don't think i am catching the Stale Element properly here. Any help would be greatly appreciated!`
pages_remaining = True
while pages_remaining:
button=driver.find_element_by_class_name("arrow-right")
href_data = button.get_attribute('href')
if href_data is not None:
soup=BeautifulSoup(driver.page_source,"html.parser")
data = soup.find_all("div",{"class":"shelfProductStamp-content row"})
count = 1
for item in data:
ProductText=item.find("a",attrs={"class":"shelfProductStamp-imageLink"})["title"]
if item.find("span",attrs={"class":"sf-pricedisplay"}) is not None:
Price=item.find("span",attrs={"class":"sf-pricedisplay"}).text
else:
Price=""
if item.find("p",attrs={"class":"sf-comparativeText"}) is not None:
SubPrice1=item.find("p",attrs={"class":"sf-comparativeText"}).text
else:
SubPrice1=""
if item.find("span",attrs={"class":"sf-regoption"}) is not None:
Option=item.find("span",attrs={"class":"sf-regoption"}).text
else:
Option=""
SubPrice=str(SubPrice1)+"-"+str(Option)
SaleDates=item.find("div",attrs={"class":"sale-dates"}).text
urll2=driver.current_url
PageNo=driver.find_element_by_class_name("current").text
writer.writerow([ProductText,Price,SubPrice,SaleDates,PageNo])
count+=1
try:
def find(driver):
element = driver.find_element_by_class_name("arrow-right")
if element:
return element
else:
pages_remaining=False
#driver.quit()
time.sleep(10)
driver.implicitly_wait(10)
element = WebDriverWait(driver, 60).until(find)
driver.execute_script("arguments[0].click();", element)
except StaleElementReferenceException:
pass
else:
break
Thanks
When you set pages_remaining = False inside the find() function, that is a local variable. It is not the same variable as pages_remaining in the outer loop.
If you want to do it that way, you'll need to make it a global.
Thanks for your help here. I managed to fix this by simply adding another if statement at the end and swapping the time.sleep(10) as below
try:
def find(driver):
element = driver.find_element_by_class_name("arrow-right")
if element:
return element
driver.implicitly_wait(10)
element = WebDriverWait(driver, 60).until(find)
driver.execute_script("arguments[0].click();", element)
time.sleep(10)
except StaleElementReferenceException:
pass
if href_data is None:
break